diff -Nru ccache-4.2.1/ARCHITECTURE.md ccache-4.5.1/ARCHITECTURE.md --- ccache-4.2.1/ARCHITECTURE.md 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/ARCHITECTURE.md 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,35 @@ +# Ccache architecture + +## Code structure + +### Top-level directories + +* `ci`: Utility scripts used in CI. +* `cmake`: CMake scripts. +* `doc`: Documentation. +* `dockerfiles`: Dockerfiles that specify different environments of interest for + ccache. +* `misc`: Miscellaneous utility scripts, example files, etc. +* `src`: Source code. See below. +* `test`: Integration test suite which tests the ccache binary in different + scenarios. +* `unittest`: Unit test suite which typically tests individual functions. + +### Subdirectories of `src` + +This section describes the directory structure that the project aims to +transform the `src` directory into in the long run to make the code base easier +to understand and work with. In other words, this is work in progress. + +* `compiler`: Knowledge about things like compiler options, compiler behavior, + preprocessor output format, etc. Ideally this code should in the future be + refactored into compiler-specific frontends, such as GCC, Clang, NVCC, MSVC, + etc. +* `compression`: Compression formats. +* `core`: Everything not part of other directories. +* `storage`: Storage backends. +* `storage/primary`: Code for the primary storage backend. +* `storage/secondary`: Code for secondary storage backends. +* `third_party`: Bundled third party code. +* `util`: Generic utility functionality that does not depend on ccache-specific + things. diff -Nru ccache-4.2.1/ci/collect-testdir ccache-4.5.1/ci/collect-testdir --- ccache-4.2.1/ci/collect-testdir 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/ci/collect-testdir 2021-11-17 19:31:58.000000000 +0000 @@ -6,7 +6,7 @@ testdir=build/testdir else echo "No testdir found" >&2 - exit 1 + exit 0 fi tar -caf testdir.tar.xz $testdir diff -Nru ccache-4.2.1/.clang-format ccache-4.5.1/.clang-format --- ccache-4.2.1/.clang-format 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/.clang-format 2021-11-17 19:31:58.000000000 +0000 @@ -1,8 +1,9 @@ -# This configuration should work with Clang-Format 6.0 and higher. +# This configuration should work with Clang-Format 10 and higher. --- Language: Cpp BasedOnStyle: LLVM +AllowAllConstructorInitializersOnNextLine: false AllowShortFunctionsOnASingleLine: None AlwaysBreakAfterReturnType: AllDefinitions AlwaysBreakBeforeMultilineStrings: true @@ -25,12 +26,18 @@ IncludeCategories: - Regex: '^"system.hpp"$' Priority: 1 - - Regex: '^"third_party/' - Priority: 3 + - Regex: '^["<]third_party/' + Priority: 4 + # System headers: + - Regex: '\.h>$' + Priority: 5 + # C++ headers: + - Regex: '^<[^.]+>$' + Priority: 6 - Regex: '^"' Priority: 2 - Regex: '.*' - Priority: 4 + Priority: 3 IndentPPDirectives: AfterHash KeepEmptyLinesAtTheStartOfBlocks: false PointerAlignment: Left diff -Nru ccache-4.2.1/cmake/CcachePackConfig.cmake ccache-4.5.1/cmake/CcachePackConfig.cmake --- ccache-4.2.1/cmake/CcachePackConfig.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/CcachePackConfig.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -1,10 +1,6 @@ # Note: This is part of CMakeLists.txt file, not to be confused with # CPackConfig.cmake. -if(${CMAKE_VERSION} VERSION_LESS "3.9") - set(CPACK_PACKAGE_DESCRIPTION "${CMAKE_PROJECT_DESCRIPTION}") -endif() - # From CcacheVersion.cmake. set(CPACK_PACKAGE_VERSION ${CCACHE_VERSION}) diff -Nru ccache-4.2.1/cmake/CcacheVersion.cmake ccache-4.5.1/cmake/CcacheVersion.cmake --- ccache-4.2.1/cmake/CcacheVersion.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/CcacheVersion.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -22,7 +22,7 @@ # CCACHE_VERSION_ORIGIN is set to "archive" in scenario 1 and "git" in scenario # 3. -set(version_info "1a07c09dcbba202572c5ab1bc4b736183a318aa7 HEAD, tag: v4.2.1, origin/master, origin/HEAD, master") +set(version_info "764dc83795d9a05c2a9d69f267ed86f61a253d09 HEAD, tag: v4.5.1, origin/HEAD, origin/4.5-maint, 4.5-maint") if(version_info MATCHES "^([0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f])[0-9a-f]* (.*)") # Scenario 1. @@ -48,7 +48,7 @@ else() macro(git) execute_process( - COMMAND "${GIT_EXECUTABLE}" ${ARGN} + COMMAND "${GIT_EXECUTABLE}" -C "${CMAKE_SOURCE_DIR}" ${ARGN} OUTPUT_VARIABLE git_stdout OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_VARIABLE git_stderr ERROR_STRIP_TRAILING_WHITESPACE) endmacro() diff -Nru ccache-4.2.1/cmake/CIBuildType.cmake ccache-4.5.1/cmake/CIBuildType.cmake --- ccache-4.2.1/cmake/CIBuildType.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/CIBuildType.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -1,18 +1,18 @@ # Add a build type called "CI" which is like RelWithDebInfo but with assertions # enabled, i.e. without passing -DNDEBUG to the compiler. -set(CMAKE_CXX_FLAGS_CI ${CMAKE_CXX_FLAGS_RELWITHDEBINFO} CACHE STRING +set(CMAKE_CXX_FLAGS_CI "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" CACHE STRING "Flags used by the C++ compiler during CI builds." FORCE) -set(CMAKE_C_FLAGS_CI ${CMAKE_C_FLAGS_RELWITHDEBINFO} CACHE STRING +set(CMAKE_C_FLAGS_CI "${CMAKE_C_FLAGS_RELWITHDEBINFO}" CACHE STRING "Flags used by the C compiler during CI builds." FORCE) set(CMAKE_EXE_LINKER_FLAGS_CI - ${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} CACHE STRING + "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO}" CACHE STRING "Flags used for linking binaries during CI builds." FORCE) set(CMAKE_SHARED_LINKER_FLAGS_CI - ${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} CACHE STRING + "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO}" CACHE STRING "Flags used by the shared libraries linker during CI builds." FORCE) mark_as_advanced( @@ -25,7 +25,7 @@ "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel CI." FORCE) -string(REGEX REPLACE "[/-]DNDEBUG" "" CMAKE_CXX_FLAGS_CI ${CMAKE_CXX_FLAGS_CI}) -string(REGEX REPLACE "[/-]DNDEBUG" "" CMAKE_C_FLAGS_CI ${CMAKE_C_FLAGS_CI}) -string(STRIP ${CMAKE_CXX_FLAGS_CI} CMAKE_CXX_FLAGS_CI) -string(STRIP ${CMAKE_C_FLAGS_CI} CMAKE_C_FLAGS_CI) +string(REGEX REPLACE "[/-]DNDEBUG" "" CMAKE_CXX_FLAGS_CI "${CMAKE_CXX_FLAGS_CI}") +string(REGEX REPLACE "[/-]DNDEBUG" "" CMAKE_C_FLAGS_CI "${CMAKE_C_FLAGS_CI}") +string(STRIP "${CMAKE_CXX_FLAGS_CI}" CMAKE_CXX_FLAGS_CI) +string(STRIP "${CMAKE_C_FLAGS_CI}" CMAKE_C_FLAGS_CI) diff -Nru ccache-4.2.1/cmake/CodeAnalysis.cmake ccache-4.5.1/cmake/CodeAnalysis.cmake --- ccache-4.2.1/cmake/CodeAnalysis.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/CodeAnalysis.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -1,38 +1,30 @@ option(ENABLE_CPPCHECK "Enable static analysis with Cppcheck" OFF) if(ENABLE_CPPCHECK) - if(${CMAKE_VERSION} VERSION_LESS "3.10") - message(WARNING "Cppcheck requires CMake 3.10") + find_program(CPPCHECK_EXE cppcheck) + mark_as_advanced(CPPCHECK_EXE) # Don't show in CMake UIs + if(CPPCHECK_EXE) + set(CMAKE_CXX_CPPCHECK + ${CPPCHECK_EXE} + --suppressions-list=${CMAKE_SOURCE_DIR}/misc/cppcheck-suppressions.txt + --inline-suppr + -q + --enable=all + --force + --std=c++14 + -I ${CMAKE_SOURCE_DIR} + --template="cppcheck: warning: {id}:{file}:{line}: {message}" + -i src/third_party) else() - find_program(CPPCHECK_EXE cppcheck) - mark_as_advanced(CPPCHECK_EXE) # Don't show in CMake UIs - if(CPPCHECK_EXE) - set(CMAKE_CXX_CPPCHECK - ${CPPCHECK_EXE} - --suppressions-list=${CMAKE_SOURCE_DIR}/misc/cppcheck-suppressions.txt - --inline-suppr - -q - --enable=all - --force - --std=c++11 - -I ${CMAKE_SOURCE_DIR} - --template="cppcheck: warning: {id}:{file}:{line}: {message}" - -i src/third_party) - else() - message(WARNING "Cppcheck requested but executable not found") - endif() + message(WARNING "Cppcheck requested but executable not found") endif() endif() option(ENABLE_CLANG_TIDY "Enable static analysis with Clang-Tidy" OFF) if(ENABLE_CLANG_TIDY) - if(${CMAKE_VERSION} VERSION_LESS "3.6") - message(WARNING "Clang-Tidy requires CMake 3.6") + find_program(CLANGTIDY clang-tidy) + if(CLANGTIDY) + set(CMAKE_CXX_CLANG_TIDY ${CLANGTIDY}) else() - find_program(CLANGTIDY clang-tidy) - if(CLANGTIDY) - set(CMAKE_CXX_CLANG_TIDY ${CLANGTIDY}) - else() - message(SEND_ERROR "Clang-Tidy requested but executable not found") - endif() + message(SEND_ERROR "Clang-Tidy requested but executable not found") endif() endif() diff -Nru ccache-4.2.1/cmake/config.h.in ccache-4.5.1/cmake/config.h.in --- ccache-4.2.1/cmake/config.h.in 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/config.h.in 2021-11-17 19:31:58.000000000 +0000 @@ -1,3 +1,6 @@ +// This file is included by all compilation units, including those in +// src/third_party. It should only contain macros and typedefs. + #pragma once #ifdef __clang__ # pragma clang diagnostic push @@ -6,6 +9,11 @@ # endif #endif +#ifdef __MINGW32__ +# define __USE_MINGW_ANSI_STDIO 1 +# define __STDC_FORMAT_MACROS 1 +#endif + // For example for vasprintf under i686-w64-mingw32-g++-posix. The later // definition of _XOPEN_SOURCE disables certain features on Linux, so we need // _GNU_SOURCE to re-enable them (makedev, tm_zone). @@ -56,8 +64,6 @@ #cmakedefine _WIN32_WINNT @_WIN32_WINNT@ // clang-format on -#define SYSCONFDIR "@CMAKE_INSTALL_FULL_SYSCONFDIR@" - #ifdef __clang__ # pragma clang diagnostic pop #endif @@ -109,6 +115,9 @@ // Define if "f_fstypename" is a member of "struct statfs". #cmakedefine HAVE_STRUCT_STATFS_F_FSTYPENAME +// Define if "st_atim" is a member of "struct stat". +#cmakedefine HAVE_STRUCT_STAT_ST_ATIM + // Define if "st_ctim" is a member of "struct stat". #cmakedefine HAVE_STRUCT_STAT_ST_CTIM @@ -174,3 +183,35 @@ # undef HAVE_STRUCT_STAT_ST_CTIM # undef HAVE_STRUCT_STAT_ST_MTIM #endif + +// Typedefs that make it possible to use common types in ccache header files +// without including core/wincompat.hpp. +#ifdef _WIN32 +# ifdef _MSC_VER +typedef unsigned __int32 mode_t; +typedef int pid_t; +# endif // _MSC_VER +#endif // _WIN32 + +// GCC version of a couple of standard C++ attributes. +#ifdef __GNUC__ +# define nodiscard gnu::warn_unused_result +# define maybe_unused gnu::unused +#endif + +// O_BINARY is needed when reading binary data on Windows, so use it everywhere +// with a compatibility define for Unix platforms. +#if !defined(_WIN32) && !defined(O_BINARY) +# define O_BINARY 0 +#endif + +#ifndef ESTALE +# define ESTALE -1 +#endif + +#define SYSCONFDIR "@CMAKE_INSTALL_FULL_SYSCONFDIR@" + +#cmakedefine INODE_CACHE_SUPPORTED + +// Buffer size for I/O operations. Should be a multiple of 4 KiB. +#define CCACHE_READ_BUFFER_SIZE 65536 diff -Nru ccache-4.2.1/cmake/DevModeWarnings.cmake ccache-4.5.1/cmake/DevModeWarnings.cmake --- ccache-4.2.1/cmake/DevModeWarnings.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/DevModeWarnings.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -82,8 +82,6 @@ CCACHE_COMPILER_WARNINGS "-Wno-zero-as-null-pointer-constant") add_compile_flag_if_supported( CCACHE_COMPILER_WARNINGS "-Wno-undefined-func-template") - add_compile_flag_if_supported( - CCACHE_COMPILER_WARNINGS "-Wno-return-std-move-in-c++11") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list( APPEND diff -Nru ccache-4.2.1/cmake/Findhiredis.cmake ccache-4.5.1/cmake/Findhiredis.cmake --- ccache-4.2.1/cmake/Findhiredis.cmake 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/cmake/Findhiredis.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,84 @@ +if(HIREDIS_FROM_INTERNET) + set(hiredis_version "1.0.2") + set(hiredis_url https://github.com/redis/hiredis/archive/v${hiredis_version}.tar.gz) + + set(hiredis_dir ${CMAKE_BINARY_DIR}/hiredis-${hiredis_version}) + set(hiredis_build ${CMAKE_BINARY_DIR}/hiredis-build) + + if(NOT EXISTS "${hiredis_dir}.tar.gz") + file(DOWNLOAD "${hiredis_url}" "${hiredis_dir}.tar.gz" STATUS download_status) + list(GET download_status 0 error_code) + if(error_code) + file(REMOVE "${hiredis_dir}.tar.gz") + list(GET download_status 1 error_message) + message(FATAL "Failed to download hiredis: ${error_message}") + endif() + endif() + + execute_process( + COMMAND tar xf "${hiredis_dir}.tar.gz" + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + RESULT_VARIABLE tar_error) + if(NOT tar_error EQUAL 0) + message(FATAL "extracting ${hiredis_dir}.tar.gz failed") + endif() + + set( + hiredis_sources + "${hiredis_dir}/alloc.c" + "${hiredis_dir}/async.c" + "${hiredis_dir}/dict.c" + "${hiredis_dir}/hiredis.c" + "${hiredis_dir}/net.c" + "${hiredis_dir}/read.c" + "${hiredis_dir}/sds.c" + "${hiredis_dir}/sockcompat.c" + ) + add_library(libhiredis_static STATIC EXCLUDE_FROM_ALL ${hiredis_sources}) + add_library(HIREDIS::HIREDIS ALIAS libhiredis_static) + if(WIN32) + target_compile_definitions(libhiredis_static PRIVATE _CRT_SECURE_NO_WARNINGS) + target_link_libraries(libhiredis_static PUBLIC ws2_32) + endif() + + make_directory("${hiredis_dir}/include") + make_directory("${hiredis_dir}/include/hiredis") + file(GLOB hiredis_headers "${hiredis_dir}/*.h") + file(COPY ${hiredis_headers} DESTINATION "${hiredis_dir}/include/hiredis") + set_target_properties( + libhiredis_static + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "$") +else() + find_package(PkgConfig) + if(PKG_CONFIG_FOUND) + pkg_check_modules(HIREDIS hiredis>=${hiredis_FIND_VERSION}) + find_library(HIREDIS_LIBRARY ${HIREDIS_LIBRARIES} HINTS ${HIREDIS_LIBDIR}) + find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h HINTS ${HIREDIS_PREFIX}/include) + else() + find_library(HIREDIS_LIBRARY hiredis) + find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h) + endif() + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args( + hiredis + "please install libhiredis or use -DHIREDIS_FROM_INTERNET=ON or disable with -DREDIS_STORAGE_BACKEND=OFF" + HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY) + mark_as_advanced(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY) + + add_library(HIREDIS::HIREDIS UNKNOWN IMPORTED) + set_target_properties( + HIREDIS::HIREDIS + PROPERTIES + IMPORTED_LOCATION "${HIREDIS_LIBRARY}" + INTERFACE_COMPILE_OPTIONS "${HIREDIS_CFLAGS_OTHER}" + INTERFACE_INCLUDE_DIRECTORIES "${HIREDIS_INCLUDE_DIR}") +endif() + +include(FeatureSummary) +set_package_properties( + hiredis + PROPERTIES + URL "https://github.com/redis/hiredis" + DESCRIPTION "Hiredis is a minimalistic C client library for the Redis database") diff -Nru ccache-4.2.1/cmake/Findzstd.cmake ccache-4.5.1/cmake/Findzstd.cmake --- ccache-4.2.1/cmake/Findzstd.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/Findzstd.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -6,7 +6,7 @@ # Although ${zstd_FIND_VERSION} was requested, let's download a newer version. # Note: The directory structure has changed in 1.3.0; we only support 1.3.0 # and newer. - set(zstd_version "1.4.9") + set(zstd_version "1.5.0") set(zstd_url https://github.com/facebook/zstd/archive/v${zstd_version}.tar.gz) set(zstd_dir ${CMAKE_BINARY_DIR}/zstd-${zstd_version}) diff -Nru ccache-4.2.1/cmake/GenerateConfigurationFile.cmake ccache-4.5.1/cmake/GenerateConfigurationFile.cmake --- ccache-4.2.1/cmake/GenerateConfigurationFile.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/GenerateConfigurationFile.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -43,9 +43,9 @@ check_function_exists(${func} ${func_var}) endforeach() -include(CheckCSourceCompiles) -set(CMAKE_REQUIRED_LINK_OPTIONS -pthread) -check_c_source_compiles( +include(CheckCXXSourceCompiles) +set(CMAKE_REQUIRED_FLAGS -pthread) +check_cxx_source_compiles( [=[ #include int main() @@ -57,15 +57,17 @@ ]=] HAVE_PTHREAD_MUTEX_ROBUST) check_function_exists(pthread_mutexattr_setpshared HAVE_PTHREAD_MUTEXATTR_SETPSHARED) -set(CMAKE_REQUIRED_LINK_OPTIONS) +set(CMAKE_REQUIRED_FLAGS) include(CheckStructHasMember) +check_struct_has_member("struct stat" st_atim sys/stat.h + HAVE_STRUCT_STAT_ST_ATIM LANGUAGE CXX) check_struct_has_member("struct stat" st_ctim sys/stat.h - HAVE_STRUCT_STAT_ST_CTIM) + HAVE_STRUCT_STAT_ST_CTIM LANGUAGE CXX) check_struct_has_member("struct stat" st_mtim sys/stat.h - HAVE_STRUCT_STAT_ST_MTIM) + HAVE_STRUCT_STAT_ST_MTIM LANGUAGE CXX) check_struct_has_member("struct statfs" f_fstypename sys/mount.h - HAVE_STRUCT_STATFS_F_FSTYPENAME) + HAVE_STRUCT_STATFS_F_FSTYPENAME LANGUAGE CXX) include(CheckCXXSourceCompiles) check_cxx_source_compiles( @@ -98,5 +100,9 @@ # alias set(MTR_ENABLED "${ENABLE_TRACING}") +if(HAVE_SYS_MMAN_H AND HAVE_PTHREAD_MUTEXATTR_SETPSHARED) + set(INODE_CACHE_SUPPORTED 1) +endif() + configure_file(${CMAKE_SOURCE_DIR}/cmake/config.h.in ${CMAKE_BINARY_DIR}/config.h @ONLY) diff -Nru ccache-4.2.1/cmake/StandardSettings.cmake ccache-4.5.1/cmake/StandardSettings.cmake --- ccache-4.2.1/cmake/StandardSettings.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/StandardSettings.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -3,10 +3,12 @@ add_library(standard_settings INTERFACE) -# Not supported in CMake 3.4: target_compile_features(project_options INTERFACE -# c_std_11 cxx_std_11) - if(CMAKE_CXX_COMPILER_ID MATCHES "^GNU|(Apple)?Clang$") + target_compile_options( + standard_settings + INTERFACE -include ${CMAKE_BINARY_DIR}/config.h + ) + option(ENABLE_COVERAGE "Enable coverage reporting for GCC/Clang" FALSE) if(ENABLE_COVERAGE) target_compile_options(standard_settings INTERFACE --coverage -O0 -g) @@ -50,5 +52,17 @@ include(StdAtomic) elseif(MSVC) - target_compile_options(standard_settings INTERFACE /std:c++latest /Zc:preprocessor /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS) + target_compile_options(standard_settings INTERFACE "/FI${CMAKE_BINARY_DIR}/config.h") + + target_compile_options( + standard_settings + INTERFACE /Zc:preprocessor /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS + ) +endif() + +if(WIN32) + target_compile_definitions( + standard_settings + INTERFACE WIN32_LEAN_AND_MEAN + ) endif() diff -Nru ccache-4.2.1/cmake/UseFastestLinker.cmake ccache-4.5.1/cmake/UseFastestLinker.cmake --- ccache-4.2.1/cmake/UseFastestLinker.cmake 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/cmake/UseFastestLinker.cmake 2021-11-17 19:31:58.000000000 +0000 @@ -1,3 +1,28 @@ +if(NOT CCACHE_DEV_MODE) + # For ccache, using a faster linker is in practice only relevant to reduce the + # compile-link-test cycle for developers, so use the standard linker for + # non-developer builds. + return() +endif() + +if(MSVC) + message(STATUS "Using standard linker for MSVC") + return() +endif() + +if(ENABLE_IPO) + message(STATUS "Using standard linker for IPO") + return() +endif() + +if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) + # Be conservative and only probe for a faster linker on platforms that likely + # don't have toolchain bugs. See for example + # . + message(STATUS "Not probing for faster linker on ${CMAKE_SYSTEM_PROCESSOR}") + return() +endif() + function(check_linker linker) string(TOUPPER ${linker} upper_linker) file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/CMakefiles/CMakeTmp/main.c" "int main() { return 0; }") @@ -15,23 +40,31 @@ return() endif() - set(use_default_linker 1) + # prefer an lld that matches the clang version + if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_VERSION MATCHES "^([0-9]*)\\.") + check_linker(lld-${CMAKE_MATCH_1}) + if(HAVE_LD_LLD-${CMAKE_MATCH_1}) + link_libraries("-fuse-ld=lld-${CMAKE_MATCH_1}") + message(STATUS "Using lld-${CMAKE_MATCH_1} linker") + return() + endif() + endif() + check_linker(lld) if(HAVE_LD_LLD) link_libraries("-fuse-ld=lld") - set(use_default_linker 0) message(STATUS "Using lld linker") - else() - check_linker(gold) - if(HAVE_LD_GOLD) - link_libraries("-fuse-ld=gold") - set(use_default_linker 0) - message(STATUS "Using gold linker") - endif() + return() endif() - if(use_default_linker) - message(STATUS "Using default linker") + + check_linker(gold) + if(HAVE_LD_GOLD) + link_libraries("-fuse-ld=gold") + message(STATUS "Using gold linker") + return() endif() + + message(STATUS "Using default linker") endfunction() option(USE_FASTER_LINKER "Use the lld or gold linker instead of the default for faster linking" TRUE) diff -Nru ccache-4.2.1/CMakeLists.txt ccache-4.5.1/CMakeLists.txt --- ccache-4.2.1/CMakeLists.txt 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4.3) +cmake_minimum_required(VERSION 3.10) project(ccache LANGUAGES C CXX) if(MSVC) @@ -8,8 +8,10 @@ endif() set(CMAKE_PROJECT_DESCRIPTION "a fast C/C++ compiler cache") -if(NOT "${CMAKE_CXX_STANDARD}") - set(CMAKE_CXX_STANDARD 11) +if(MSVC) + set(CMAKE_CXX_STANDARD 17) # Need support for std::filesystem +else() + set(CMAKE_CXX_STANDARD 14) endif() set(CMAKE_CXX_STANDARD_REQUIRED YES) set(CMAKE_CXX_EXTENSIONS NO) @@ -28,10 +30,8 @@ # C++ error messages) # -# Clang 3.4 and AppleClang 6.0 fail to compile doctest. -# GCC 4.8.4 is known to work OK but give warnings. -if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.5) - OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8.4) +if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.8) + OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5) OR (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0)) message( FATAL_ERROR @@ -39,10 +39,8 @@ "You need one listed here: https://ccache.dev/platform-compiler-language-support.html") endif() -# All Clang problems / special handling ccache has are because of version 3.5. -# All GCC problems / special handling ccache has are because of version 4.8.4. -if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.6) - OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) +if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4) + OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6)) message( WARNING "The compiler you are using is rather old.\n" @@ -58,30 +56,27 @@ # include(CcacheVersion) -if("${CCACHE_VERSION_ORIGIN}" STREQUAL git OR DEFINED ENV{CI}) - set(CCACHE_DEV_MODE ON) -else() - set(CCACHE_DEV_MODE OFF) +if(NOT DEFINED CCACHE_DEV_MODE) + if("${CCACHE_VERSION_ORIGIN}" STREQUAL git OR DEFINED ENV{CI}) + set(CCACHE_DEV_MODE ON) + else() + set(CCACHE_DEV_MODE OFF) + endif() endif() message(STATUS "Ccache dev mode: ${CCACHE_DEV_MODE}") -include(UseCcache) -if(NOT MSVC) - include(UseFastestLinker) +option(ENABLE_IPO "Enable interprocedural (link time, LTO) optimization" OFF) +if(ENABLE_IPO AND NOT MINGW) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) endif() + +include(UseCcache) +include(UseFastestLinker) include(StandardSettings) include(StandardWarnings) include(CIBuildType) include(DefaultBuildType) -if(NOT ${CMAKE_VERSION} VERSION_LESS "3.9") - cmake_policy(SET CMP0069 NEW) - option(ENABLE_IPO "Enable interprocedural (link time, LTO) optimization" OFF) - if(ENABLE_IPO) - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) - endif() -endif() - # # Configuration # @@ -89,23 +84,27 @@ include(GenerateConfigurationFile) include(GenerateVersionFile) -if(HAVE_SYS_MMAN_H AND HAVE_PTHREAD_MUTEXATTR_SETPSHARED) - set(INODE_CACHE_SUPPORTED 1) -endif() - # # Third party # +set(HIREDIS_FROM_INTERNET_DEFAULT OFF) set(ZSTD_FROM_INTERNET_DEFAULT OFF) # Default to downloading deps for Visual Studio, unless using a package manager. if(MSVC AND NOT CMAKE_TOOLCHAIN_FILE MATCHES "vcpkg|conan") + set(HIREDIS_FROM_INTERNET_DEFAULT ON) set(ZSTD_FROM_INTERNET_DEFAULT ON) endif() option(ZSTD_FROM_INTERNET "Download and use libzstd from the Internet" ${ZSTD_FROM_INTERNET_DEFAULT}) find_package(zstd 1.1.2 REQUIRED) +option(REDIS_STORAGE_BACKEND "Enable Redis secondary storage" ON) +if(REDIS_STORAGE_BACKEND) + option(HIREDIS_FROM_INTERNET "Download and use libhiredis from the Internet" ${HIREDIS_FROM_INTERNET_DEFAULT}) + find_package(hiredis 0.13.3 REQUIRED) +endif() + # # Special flags # @@ -128,12 +127,15 @@ # ccache executable # add_executable(ccache src/main.cpp) -target_link_libraries(ccache PRIVATE standard_settings standard_warnings ccache_lib) +target_link_libraries(ccache PRIVATE standard_settings standard_warnings ccache_framework) # # Documentation # -add_subdirectory(doc) +option(ENABLE_DOCUMENTATION "Enable documentation" ON) +if(ENABLE_DOCUMENTATION) + add_subdirectory(doc) +endif() # # Installation @@ -176,26 +178,16 @@ # # Special formatting targets # -find_program( - CLANG_FORMAT_EXE - NAMES "clang-format" - DOC "Path to clang-format executable.") -mark_as_advanced(CLANG_FORMAT_EXE) # Don't show in CMake UIs - -if(NOT CLANG_FORMAT_EXE) - message(STATUS "clang-format not found") -else() - add_custom_target( - format - COMMAND misc/format-files --all - COMMENT "Formatting code" - USES_TERMINAL - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - - add_custom_target( - check_format - COMMAND misc/format-files --all --check - COMMENT "Checking code formatting" - USES_TERMINAL - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -endif() +add_custom_target( + format + COMMAND misc/format-files --all + COMMENT "Formatting code" + USES_TERMINAL + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +add_custom_target( + check_format + COMMAND misc/format-files --all --check + COMMENT "Checking code formatting" + USES_TERMINAL + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) diff -Nru ccache-4.2.1/CONTRIBUTING.md ccache-4.5.1/CONTRIBUTING.md --- ccache-4.2.1/CONTRIBUTING.md 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/CONTRIBUTING.md 2021-11-17 19:31:58.000000000 +0000 @@ -33,6 +33,7 @@ Here are some hints to make the process smoother: +* Have a look in `ARCHITECTURE.md` for an overview of the source code tree. * If you plan to implement major changes it is wise to open an issue on GitHub (or ask in the Gitter room, or send a mail to the mailing list) asking for comments on your plans before doing the bulk of the work. That way you can @@ -42,33 +43,59 @@ for merging yet but you want early comments and CI test results? Then create a draft pull request as described in [this Github blog post](https://github.blog/2019-02-14-introducing-draft-pull-requests/). +* Please add test cases for your new changes if applicable. * Please follow the ccache's code style (see the section below). -* Consider [A Note About Git Commit - Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) - when writing commit messages. -## Code style +## Commit message conventions + +It is preferable, but not mandatory, to format commit messages in the spirit of +[Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). The +commit message subject should look like this: + + : + (): + +`` is a succinct description of the change: + +* Use the imperative, present tense: "Change", not "Changed" nor "Changes". +* Capitalize the first letter. +* No dot (`.`) at the end. + +Here is a summary of types used for ccache: + +| Type | Explanation | +| ------------ | ----------- | +| **build** | A change of the build system or build configuration. | +| **bump** | An increase of the version of an external dependency or an update of a bundled third party package. | +| **chore** | A change that doesn't match any other type. | +| **ci** | A change of CI scripts or configuration. | +| **docs** | A change of documentation only. | +| **enhance** | An enhancement of the code without adding a user-visible feature, for example adding a new utility class to be used by a future feature or refactoring. | +| **feat** | An addition or improvement of a user-visible feature. | +| **fix** | A bug fix (not necessarily user-visible). | +| **perf** | A performance improvement. | +| **refactor** | A restructuring of the existing code without changing its external behavior. | +| **style** | A change of code style. | +| **test** | An addition or modification of tests or test framework. | -Ccache was written in C99 until 2019 when it started being converted to C++11. -The conversion is a slow work in progress, which is why there is some C-style -code left. Please refrain from doing large C to C++ conversions; do it little by -little. +## Code style Source code formatting is defined by `.clang-format` in the root directory. The format is loosely based on [LLVM's code formatting -style](https://llvm.org/docs/CodingStandards.html) with some exceptions. It's -highly recommended to install -[Clang-Format](https://clang.llvm.org/docs/ClangFormat.html) 6.0 or newer and -run `make format` to format changes according to ccache's code style. Or even -better: set up your editor to run Clang-Format automatically when saving. If you -don't run Clang-Format then the ccache authors have to do it for you. +style](https://llvm.org/docs/CodingStandards.html) with some exceptions. Run +`make format` (or `ninja format` if you use Ninja) to format changes according +to ccache's code style. Or even better: set up your editor to run +`/misc/clang-format` (or any other Clang-Format version 10 +binary) automatically when saving. Newer Clang-Format versions likely also work +fine. Please follow these conventions: -* Use `UpperCamelCase` for types (e.g. classes and structs) and namespaces. -* Use `UPPER_CASE` names for macros and (non-class )enum values. -* Use `snake_case` for other names (functions, variables, enum class values, - etc.). +* Use `UpperCamelCase` for types (e.g. classes and structs). +* Use `UPPER_CASE` names for macros and (non-class) enum values. +* Use `snake_case` for other names (namespaces, functions, variables, enum class + values, etc.). (Namespaces used to be in `UpperCamelCase`; transition is work + in progress.) * Use an `m_` prefix for non-public member variables. * Use a `g_` prefix for global mutable variables. * Use a `k_` prefix for global constants. diff -Nru ccache-4.2.1/debian/changelog ccache-4.5.1/debian/changelog --- ccache-4.2.1/debian/changelog 2021-08-10 01:09:19.000000000 +0000 +++ ccache-4.5.1/debian/changelog 2021-11-17 19:59:25.000000000 +0000 @@ -1,14 +1,43 @@ -ccache (4.2.1-0ubuntu1) impish; urgency=medium +ccache (4.5.1-1) unstable; urgency=medium - * New upstream version, fixing build with glibc 2.34. + * New upstream release 4.5.1 - -- Michael Hudson-Doyle Tue, 10 Aug 2021 13:09:19 +1200 + -- Joel Rosdahl Wed, 17 Nov 2021 20:59:25 +0100 -ccache (4.2-1build1) hirsute; urgency=medium +ccache (4.5-1) unstable; urgency=medium - * No change rebuild with fixed ownership. + * Refer to common license file for CC0-1.0 + * New upstream release 4.5 - -- Dimitri John Ledkov Tue, 16 Feb 2021 15:10:38 +0000 + -- Joel Rosdahl Sat, 13 Nov 2021 16:05:58 +0100 + +ccache (4.4.2-1) unstable; urgency=high + + * New upstream release 4.4.2, with an important bug fix of a serious + regression in ccache 4.4 where ccache could produce false direct cache + hits in some situations if it decides to disable the direct mode + temporarily (e.g. due to “too new header” file) + * Remove now obsolete upstream patch for failing .incbin test + + -- Joel Rosdahl Wed, 29 Sep 2021 08:39:12 +0200 + +ccache (4.4-1) unstable; urgency=medium + + [ Gregor Jasny ] + * Add Salsa CI build pipeline config + * Add autopkgtest + * Fix blhc CPPFLAGS warnings + * Exclude time variation due to test failures + + [ Joel Rosdahl ] + * New upstream release 4.4 + * Bump Standards-Version to 4.6.0 with no changes + * Update Build-Depends to match ccache 4.4 requirements + * Update debian_specific_usage_docs.patch to match new manual + * Apply upstream patch for failing .incbin test with newer binutils + * Update debian/copyright to match new release + + -- Joel Rosdahl Fri, 20 Aug 2021 21:44:19 +0200 ccache (4.2-1) unstable; urgency=medium diff -Nru ccache-4.2.1/debian/control ccache-4.5.1/debian/control --- ccache-4.2.1/debian/control 2021-08-10 01:07:22.000000000 +0000 +++ ccache-4.5.1/debian/control 2021-11-17 19:59:25.000000000 +0000 @@ -1,10 +1,9 @@ Source: ccache Section: devel Priority: optional -Maintainer: Ubuntu Developers -XSBC-Original-Maintainer: Joel Rosdahl -Build-Depends: debhelper-compat (= 13), zlib1g-dev, dpkg-dev (>= 1.16.1~), elfutils, cmake (>= 3.5), libzstd-dev (>= 1.1.2), asciidoc, docbook-xml, docbook-xsl -Standards-Version: 4.5.1 +Maintainer: Joel Rosdahl +Build-Depends: debhelper-compat (= 13), dpkg-dev (>= 1.16.1~), elfutils, cmake (>= 3.10), libzstd-dev (>= 1.1.2), libhiredis-dev (>= 0.14.1), asciidoctor +Standards-Version: 4.6.0 Homepage: https://ccache.dev Vcs-Browser: https://salsa.debian.org/debian/ccache Vcs-Git: https://salsa.debian.org/debian/ccache.git diff -Nru ccache-4.2.1/debian/copyright ccache-4.5.1/debian/copyright --- ccache-4.2.1/debian/copyright 2021-08-10 01:07:22.000000000 +0000 +++ ccache-4.5.1/debian/copyright 2021-11-17 19:59:25.000000000 +0000 @@ -5,14 +5,14 @@ Files: * Copyright: 2002-2007 Andrew Tridgell - 2009-2020 Joel Rosdahl + 2009-2021 Joel Rosdahl License: GPL-3+ Files: debian/* Copyright: 2002-2003 Paul Russell 2004-2009 Francois Marier 2009-2010 Y Giridhar Appaji Nag - 2010-2020 Joel Rosdahl + 2010-2021 Joel Rosdahl License: GPL-3+ Files: src/third_party/base32hex.* @@ -21,18 +21,18 @@ Files: src/third_party/blake3/* Copyright: 2019-2020 Samuel Neves and Jack O'Connor -License: CC0 or Apache +License: CC0-1.0 or Apache Files: src/third_party/doctest.h -Copyright: 2016-2019 Viktor Kirilov +Copyright: 2016-2021 Viktor Kirilov License: Expat Files: src/third_party/fmt/*.h -Copyright: 2012-2020 Victor Zverovich +Copyright: 2012-2021 Victor Zverovich License: Expat Files: src/third_party/format.cpp -Copyright: 2012-2020 Victor Zverovich +Copyright: 2012-2021 Victor Zverovich License: Expat Files: src/third_party/getopt_long.* @@ -40,10 +40,18 @@ 2003 PostgreSQL Global Development Group License: BSD-3-clause +Files: src/third_party/httplib.* +Copyright: 2021 yhirose +License: Expat + Files: src/third_party/minitrace.* Copyright: 2014 Henrik Rydgård License: Expat +Files: src/third_party/nonstd/expected.hpp +Copyright: 2016-2018 Martin Moene +License: Boost + Files: src/third_party/nonstd/optional.hpp Copyright: 2014-2018 Martin Moene License: Boost @@ -52,10 +60,23 @@ Copyright: 2017-2020 Martin Moene License: Boost +Files: src/third_party/url.* +Copyright: 2015 Christophe Meessen +License: Expat + Files: src/third_party/win32/getopt.* Copyright: 2011 Ludvik Jerabek License: LGPL-3 +Files: src/third_party/win32/mktemp.* +Copyright: 1996-2008 Theo de Raadt + 1997-2009 Todd C. Miller +License: ISC + +Files: src/third_party/win32/winerror_to_errno.h +Copyright: None specified +License: Python + Files: src/third_party/xxh* Copyright: 2012-2020 Yann Collet License: BSD-2-clause @@ -305,128 +326,16 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -License: CC0 - Creative Commons Legal Code +License: CC0-1.0 + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. . - CC0 1.0 Universal + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see . . - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - . - Statement of Purpose - . - The laws of most jurisdictions throughout the world automatically confer - exclusive Copyright and Related Rights (defined below) upon the creator - and subsequent owner(s) (each and all, an "owner") of an original work of - authorship and/or a database (each, a "Work"). - . - Certain owners wish to permanently relinquish those rights to a Work for - the purpose of contributing to a commons of creative, cultural and - scientific works ("Commons") that the public can reliably and without fear - of later claims of infringement build upon, modify, incorporate in other - works, reuse and redistribute as freely as possible in any form whatsoever - and for any purposes, including without limitation commercial purposes. - These owners may contribute to the Commons to promote the ideal of a free - culture and the further production of creative, cultural and scientific - works, or to gain reputation or greater distribution for their Work in - part through the use and efforts of others. - . - For these and/or other purposes and motivations, and without any - expectation of additional consideration or compensation, the person - associating CC0 with a Work (the "Affirmer"), to the extent that he or she - is an owner of Copyright and Related Rights in the Work, voluntarily - elects to apply CC0 to the Work and publicly distribute the Work under its - terms, with knowledge of his or her Copyright and Related Rights in the - Work and the meaning and intended legal effect of CC0 on those rights. - . - 1. Copyright and Related Rights. A Work made available under CC0 may be - protected by copyright and related or neighboring rights ("Copyright and - Related Rights"). Copyright and Related Rights include, but are not - limited to, the following: - . - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); - iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and - vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - . - 2. Waiver. To the greatest extent permitted by, but not in contravention - of, applicable law, Affirmer hereby overtly, fully, permanently, - irrevocably and unconditionally waives, abandons, and surrenders all of - Affirmer's Copyright and Related Rights and associated claims and causes - of action, whether now known or unknown (including existing as well as - future claims and causes of action), in the Work (i) in all territories - worldwide, (ii) for the maximum duration provided by applicable law or - treaty (including future time extensions), (iii) in any current or future - medium and for any number of copies, and (iv) for any purpose whatsoever, - including without limitation commercial, advertising or promotional - purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each - member of the public at large and to the detriment of Affirmer's heirs and - successors, fully intending that such Waiver shall not be subject to - revocation, rescission, cancellation, termination, or any other legal or - equitable action to disrupt the quiet enjoyment of the Work by the public - as contemplated by Affirmer's express Statement of Purpose. - . - 3. Public License Fallback. Should any part of the Waiver for any reason - be judged legally invalid or ineffective under applicable law, then the - Waiver shall be preserved to the maximum extent permitted taking into - account Affirmer's express Statement of Purpose. In addition, to the - extent the Waiver is so judged Affirmer hereby grants to each affected - person a royalty-free, non transferable, non sublicensable, non exclusive, - irrevocable and unconditional license to exercise Affirmer's Copyright and - Related Rights in the Work (i) in all territories worldwide, (ii) for the - maximum duration provided by applicable law or treaty (including future - time extensions), (iii) in any current or future medium and for any number - of copies, and (iv) for any purpose whatsoever, including without - limitation commercial, advertising or promotional purposes (the - "License"). The License shall be deemed effective as of the date CC0 was - applied by Affirmer to the Work. Should any part of the License for any - reason be judged legally invalid or ineffective under applicable law, such - partial invalidity or ineffectiveness shall not invalidate the remainder - of the License, and in such case Affirmer hereby affirms that he or she - will not (i) exercise any of his or her remaining Copyright and Related - Rights in the Work or (ii) assert any associated claims and causes of - action with respect to the Work, in either case contrary to Affirmer's - express Statement of Purpose. - . - 4. Limitations and Disclaimers. - . - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. + On Debian systems, the full text of the CC0 1.0 Universal license can be found + in the file "/usr/share/common-licenses/CC0-1.0". License: Expat Permission is hereby granted, free of charge, to any person obtaining a copy @@ -483,3 +392,65 @@ . On Debian systems, the full text of the GNU General Public License version 3 can be found in the file "/usr/share/common-licenses/GPL-3". + +License: ISC + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + . + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +License: Python + PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 + . + 1. This LICENSE AGREEMENT is between the Python Software Foundation + ("PSF"), and the Individual or Organization ("Licensee") accessing and + otherwise using this software ("Python") in source or binary form and + its associated documentation. + . + 2. Subject to the terms and conditions of this License Agreement, PSF hereby + grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, + analyze, test, perform and/or display publicly, prepare derivative works, + distribute, and otherwise use Python alone or in any derivative version, + provided, however, that PSF's License Agreement and PSF's notice of copyright, + i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Python Software Foundation; + All Rights Reserved" are retained in Python alone or in any derivative version + prepared by Licensee. + . + 3. In the event Licensee prepares a derivative work that is based on + or incorporates Python or any part thereof, and wants to make + the derivative work available to others as provided herein, then + Licensee hereby agrees to include in any such work a brief summary of + the changes made to Python. + . + 4. PSF is making Python available to Licensee on an "AS IS" + basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR + IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND + DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS + FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT + INFRINGE ANY THIRD PARTY RIGHTS. + . + 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS + A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, + OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + . + 6. This License Agreement will automatically terminate upon a material + breach of its terms and conditions. + . + 7. Nothing in this License Agreement shall be deemed to create any + relationship of agency, partnership, or joint venture between PSF and + Licensee. This License Agreement does not grant permission to use PSF + trademarks or trade name in a trademark sense to endorse or promote + products or services of Licensee, or any third party. + . + 8. By copying, installing or otherwise using Python, Licensee + agrees to be bound by the terms and conditions of this License + Agreement. diff -Nru ccache-4.2.1/debian/patches/debian_specific_usage_docs.patch ccache-4.5.1/debian/patches/debian_specific_usage_docs.patch --- ccache-4.2.1/debian/patches/debian_specific_usage_docs.patch 2021-08-10 01:07:22.000000000 +0000 +++ ccache-4.5.1/debian/patches/debian_specific_usage_docs.patch 2021-11-17 19:59:25.000000000 +0000 @@ -2,18 +2,19 @@ Author: Joel Rosdahl Forwarded: not-needed ---- a/doc/MANUAL.adoc -+++ b/doc/MANUAL.adoc -@@ -44,16 +44,17 @@ most convenient if you just want to try out ccache or wish to use it for some - specific projects. The second method is most useful for when you wish to use - ccache for all your compilations. +Index: ccache/doc/MANUAL.adoc +=================================================================== +--- ccache.orig/doc/MANUAL.adoc 2021-08-20 08:31:00.795877280 +0200 ++++ ccache/doc/MANUAL.adoc 2021-08-20 08:31:04.703813172 +0200 +@@ -38,14 +38,17 @@ --To use the first method, just make sure that *ccache* is in your *PATH*. -+To use the second method on a Debian system, it's easiest to just prepend -+`/usr/lib/ccache` to your *PATH*. `/usr/lib/ccache` contains symlinks for all -+compilers currently installed as Debian packages. + To use the first method, just make sure that `ccache` is in your `PATH`. -To use the symlinks method, do something like this: ++To use the second method on a Debian system, it's easiest to just prepend ++`/usr/lib/ccache` to your `PATH`. `/usr/lib/ccache` contains symlinks for all ++compilers currently installed as Debian packages. ++ +Alternatively, you can create any symlinks you like yourself like this: ------------------------------------------------------------------------------- diff -Nru ccache-4.2.1/debian/patches/series ccache-4.5.1/debian/patches/series --- ccache-4.2.1/debian/patches/series 2021-08-10 01:07:22.000000000 +0000 +++ ccache-4.5.1/debian/patches/series 2021-11-17 19:59:25.000000000 +0000 @@ -1,2 +1 @@ debian_specific_usage_docs.patch - diff -Nru ccache-4.2.1/debian/rules ccache-4.5.1/debian/rules --- ccache-4.2.1/debian/rules 2021-08-10 01:07:22.000000000 +0000 +++ ccache-4.5.1/debian/rules 2021-11-17 19:59:25.000000000 +0000 @@ -1,5 +1,6 @@ #!/usr/bin/make -f +export ASMFLAGS:=$(shell dpkg-buildflags --get CPPFLAGS) export DEB_BUILD_MAINT_OPTIONS = hardening=+all %: diff -Nru ccache-4.2.1/debian/salsa-ci.yml ccache-4.5.1/debian/salsa-ci.yml --- ccache-4.2.1/debian/salsa-ci.yml 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/debian/salsa-ci.yml 2021-11-17 19:59:25.000000000 +0000 @@ -0,0 +1,11 @@ +# For more information on what jobs are run see: +# https://salsa.debian.org/salsa-ci-team/pipeline +--- +include: + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml + +variables: + #SALSA_CI_LINTIAN_FAIL_WARNING: 0 + SALSA_CI_DISABLE_LINTIAN: 1 + SALSA_CI_REPROTEST_ARGS: --variations=-time diff -Nru ccache-4.2.1/debian/tests/control ccache-4.5.1/debian/tests/control --- ccache-4.2.1/debian/tests/control 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/debian/tests/control 2021-11-17 19:59:25.000000000 +0000 @@ -0,0 +1,2 @@ +Tests: run-testsuites +Depends: binutils, ccache, elfutils, gcc, libc-dev diff -Nru ccache-4.2.1/debian/tests/run-testsuites ccache-4.5.1/debian/tests/run-testsuites --- ccache-4.2.1/debian/tests/run-testsuites 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/debian/tests/run-testsuites 2021-11-17 19:59:25.000000000 +0000 @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +set -eu + +export CCACHE=/usr/bin/ccache + +TEST_SUITE_RUNNER="${PWD}/test/run" + +cd "${AUTOPKGTEST_TMP}" +"${TEST_SUITE_RUNNER}" diff -Nru ccache-4.2.1/doc/AUTHORS.adoc ccache-4.5.1/doc/AUTHORS.adoc --- ccache-4.2.1/doc/AUTHORS.adoc 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/doc/AUTHORS.adoc 2021-11-17 19:31:58.000000000 +0000 @@ -1,5 +1,4 @@ -Ccache authors -============== += Ccache authors Ccache was originally written by Andrew Tridgell and is currently developed and maintained by Joel Rosdahl. @@ -8,6 +7,7 @@ * Abubakar Nur Khalil * Aleksander Salwa +* Alexander Falkenstern * Alexander Korsunsky * Alexander Lanin * Alexey Tourbin @@ -22,6 +22,7 @@ * Arne Hasselbring * Azat Khuzhin * Bernhard Bauer +* Bin Li * Björn Jacke * Breno Guimaraes * Chiaki Ishikawa @@ -58,10 +59,12 @@ * Jon Petrissans * Jørgen P. Tjernø * Josh Soref +* Josh Triplett * Justin Lebar * Ka Ho Ng * Karl Chen * Khem Raj +* Kira Bruneau * Kona Blend * Kovarththanan Rajaratnam * Lalit Chhabra @@ -78,6 +81,7 @@ * Matthias Kretz * Matt Whitlock * Melven Roehrig-Zoellner +* Michael Kruse * Michael Marineau * Michael Meeks * Michał Mirosław @@ -106,6 +110,8 @@ * Pawel Krysiak * Per Nordlöw * Peter Budai +* Peter Steinberger +* Petr Štetiar * Philippe Proulx * Philipp Storz * Rafael Kitover @@ -113,8 +119,10 @@ * Robert Yang * Robin H. Johnson * Rolf Bjarne Kvinge +* R. Voggenauer * RW * Ryan Brown +* Ryan Burns * Ryan Egesdahl * Sam Gross * Sergei Trofimovich @@ -127,6 +135,7 @@ * Tim Potter * Tomasz Miąsko * Tom Hughes +* Tom Stellard * Tor Arne Vestbø * Vadim Petrochenkov * Ville Skyttä diff -Nru ccache-4.2.1/doc/ccache-doc.css ccache-4.5.1/doc/ccache-doc.css --- ccache-4.2.1/doc/ccache-doc.css 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/doc/ccache-doc.css 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,50 @@ +@import url(https://fonts.googleapis.com/css?family=Montserrat|Open+Sans); +@import url(https://cdn.jsdelivr.net/gh/asciidoctor/asciidoctor@2.0/data/stylesheets/asciidoctor-default.css); /* Default asciidoc style framework - important */ + +:root{ +--maincolor:#ffffff; +--primarycolor:#294172; +--secondarycolor:#5f646c; +--tertiarycolor:#cccccc; +--highlightcolor:#f5f5f5; +--sidebarbackground:#f5f5f5; +--linkcolor:#02a; +--linkcoloralternate:#db3279; +--white:#ffffff; +--black:#000000; +} + +/* Text styles */ + +body{font-family: "Open Sans", sans-serif;background-color: var(--maincolor);color:var(--black);} + +h1{color:var(--primarycolor) !important;font-family:"Montserrat",sans-serif;} +h2,h3,h4,h5,h6{color:var(--secondarycolor) !important;font-family:"Montserrat",sans-serif;} +.title{color:var(--black) !important;font-family:"Open Sans",sans-serif;font-style: normal; font-weight: normal;} +a{text-decoration: none;} +a:hover{color: #3958da; text-decoration: underline;} +p{font-family: "Open Sans",sans-serif ! important} +#toc.toc2 a:link{color:var(--linkcolor);} +blockquote{color:var(--linkcoloralternate) !important} +.quoteblock blockquote:before{color:var(--linkcoloralternate)} +code{color:var(--black);background-color: var(--highlightcolor) !important} +mark{background-color: var(--highlightcolor)} /* Text highlighting color */ + +/* Table styles */ +th{background-color: var(--maincolor);color:var(--black) !important;} +td{background-color: var(--maincolor);color: var(--black) !important} + + +#toc.toc2{background-color:var(--sidebarbackground);} +#toctitle{color:var(--black);} + +/* Responsiveness fixes */ +video { + max-width: 100%; +} + +@media all and (max-width: 600px) { +table { + width: 55vw!important; + font-size: 3vw; +} diff -Nru ccache-4.2.1/doc/CMakeLists.txt ccache-4.5.1/doc/CMakeLists.txt --- ccache-4.2.1/doc/CMakeLists.txt 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/doc/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -1,70 +1,54 @@ -find_program(ASCIIDOC_EXE asciidoc) -mark_as_advanced(ASCIIDOC_EXE) # Don't show in CMake UIs +find_program(ASCIIDOCTOR_EXE asciidoctor) +mark_as_advanced(ASCIIDOCTOR_EXE) # Don't show in CMake UIs -if(NOT ASCIIDOC_EXE) - message(WARNING "Could not find asciidoc; documentation will not be generated") +if(NOT ASCIIDOCTOR_EXE) + message(WARNING "Could not find asciidoctor; documentation will not be generated") else() - # - # HTML documentation - # - function(generate_html adoc_file) + function(generate_doc backend adoc_file output_file) get_filename_component(base_name "${adoc_file}" NAME_WE) - set(html_file "${base_name}.html") add_custom_command( - OUTPUT "${html_file}" + OUTPUT "${output_file}" COMMAND - ${ASCIIDOC_EXE} - -o "${html_file}" + ${ASCIIDOCTOR_EXE} + -o "${output_file}" -a revnumber="${CCACHE_VERSION}" - -a toc - -b xhtml11 + -a icons=font + -a toc=left + -a sectanchors + -a stylesheet="${CMAKE_CURRENT_SOURCE_DIR}/ccache-doc.css" + -b "${backend}" "${CMAKE_SOURCE_DIR}/${adoc_file}" MAIN_DEPENDENCY "${CMAKE_SOURCE_DIR}/${adoc_file}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/ccache-doc.css" ) - set(html_files "${html_files}" "${html_file}" PARENT_SCOPE) + set(doc_files "${doc_files}" "${output_file}" PARENT_SCOPE) endfunction() - generate_html(LICENSE.adoc) - generate_html(doc/AUTHORS.adoc) - generate_html(doc/MANUAL.adoc) - generate_html(doc/NEWS.adoc) - - add_custom_target(doc-html DEPENDS "${html_files}") - set(doc_files "${html_files}") + # + # HTML documentation + # + generate_doc(html LICENSE.adoc LICENSE.html) + generate_doc(html doc/AUTHORS.adoc AUTHORS.html) + generate_doc(html doc/MANUAL.adoc MANUAL.html) + generate_doc(html doc/NEWS.adoc NEWS.html) + add_custom_target(doc-html DEPENDS "${doc_files}") # # Man page # - find_program(A2X_EXE a2x) - mark_as_advanced(A2X_EXE) # Don't show in CMake UIs - if(NOT A2X_EXE) - message(WARNING "Could not find a2x; man page will not be generated") - else() - # MANUAL.adoc -> MANUAL.xml -> man page - add_custom_command( - OUTPUT MANUAL.xml - COMMAND - ${ASCIIDOC_EXE} - -o - - -a revnumber=${CCACHE_VERSION} - -d manpage - -b docbook "${CMAKE_SOURCE_DIR}/doc/MANUAL.adoc" - | perl -pe 's!\(.*?\)!\\1!g' - >MANUAL.xml - MAIN_DEPENDENCY "${CMAKE_SOURCE_DIR}/doc/MANUAL.adoc" - ) - add_custom_command( - OUTPUT ccache.1 - COMMAND ${A2X_EXE} --doctype manpage --format manpage MANUAL.xml - MAIN_DEPENDENCY MANUAL.xml - ) - add_custom_target(doc-man-page DEPENDS ccache.1) - install( - FILES "${CMAKE_CURRENT_BINARY_DIR}/ccache.1" - DESTINATION "${CMAKE_INSTALL_MANDIR}/man1" - ) - set(doc_files "${doc_files}" ccache.1) - endif() + generate_doc(manpage doc/MANUAL.adoc ccache.1.tmp) + add_custom_command( + OUTPUT ccache.1 + # Convert monospace to bold since that's typically rendered better when + # viewing the man page. + COMMAND perl -pe "'s!\\\\f\\(CR(.*?)\\\\fP!\\\\fB\\1\\\\fP!g'" ccache.1.tmp >ccache.1 + MAIN_DEPENDENCY ccache.1.tmp + ) + install( + FILES "${CMAKE_CURRENT_BINARY_DIR}/ccache.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1" + ) + add_custom_target(doc-man-page DEPENDS ccache.1) - add_custom_target(doc ALL DEPENDS "${doc_files}") + add_custom_target(doc ALL DEPENDS doc-html doc-man-page) endif() diff -Nru ccache-4.2.1/doc/INSTALL.md ccache-4.5.1/doc/INSTALL.md --- ccache-4.2.1/doc/INSTALL.md 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/doc/INSTALL.md 2021-11-17 19:31:58.000000000 +0000 @@ -6,14 +6,15 @@ To build ccache you need: -- CMake 3.4.3 or newer. -- A C++11 compiler. See [Supported platforms, compilers and +- CMake 3.10 or newer. +- A C++14 compiler. See [Supported platforms, compilers and languages](https://ccache.dev/platform-compiler-language-support.html) for details. - A C99 compiler. - [libzstd](http://www.zstd.net). If you don't have libzstd installed and can't or don't want to install it in a standard system location, there are two options: + 1. Install zstd in a custom path and set `CMAKE_PREFIX_PATH` to it, e.g. by passing `-DCMAKE_PREFIX_PATH=/some/custom/path` to `cmake`, or 2. Pass `-DZSTD_FROM_INTERNET=ON` to `cmake` to make it download libzstd @@ -24,13 +25,20 @@ Optional: +- [hiredis](https://github.com/redis/hiredis) for the Redis storage backend. If + you don't have libhiredis installed and can't or don't want to install it in a + standard system location, there are two options: + + 1. Install hiredis in a custom path and set `CMAKE_PREFIX_PATH` to it, e.g. + by passing `-DCMAKE_PREFIX_PATH=/some/custom/path` to `cmake`, or + 2. Pass `-DHIREDIS_FROM_INTERNET=ON` to `cmake` to make it download hiredis + from the Internet and unpack it in the local binary tree. Ccache will + then be linked statically to the locally built libhiredis. + + To link libhiredis statically you can use + `-DHIREDIS_LIBRARY=/path/to/libhiredis.a`. - GNU Bourne Again SHell (bash) for tests. -- [AsciiDoc](https://www.methods.co.nz/asciidoc/) to build the HTML - documentation. - - Tip: On Debian-based systems (e.g. Ubuntu), install the `docbook-xml` and - `docbook-xsl` packages in addition to `asciidoc`. Without the former the - man page generation will be very slow. -- [xsltproc](http://xmlsoft.org/XSLT/xsltproc2.html) to build the man page. +- [Asciidoctor](https://asciidoctor.org) to build the HTML documentation. - [Python](https://www.python.org) to debug and run the performance test suite. diff -Nru ccache-4.2.1/doc/MANUAL.adoc ccache-4.5.1/doc/MANUAL.adoc --- ccache-4.2.1/doc/MANUAL.adoc 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/doc/MANUAL.adoc 2021-11-17 19:31:58.000000000 +0000 @@ -1,18 +1,12 @@ -CCACHE(1) -========= -:man source: ccache -:man version: {revnumber} -:man manual: ccache Manual += ccache(1) +:mansource: Ccache {revnumber} - -Name ----- +== Name ccache - a fast C/C++ compiler cache -Synopsis --------- +== Synopsis [verse] *ccache* [_options_] @@ -20,8 +14,7 @@ _compiler_ [_compiler options_] (via symbolic link) -Description ------------ +== Description Ccache is a compiler cache. It speeds up recompilation by caching the result of previous compilations and detecting when the same compilation is being done @@ -30,21 +23,20 @@ Ccache has been carefully written to always produce exactly the same compiler output that you would get without the cache. The only way you should be able to tell that you are using ccache is the speed. Currently known exceptions to this -goal are listed under <<_caveats,CAVEATS>>. If you discover an undocumented case -where ccache changes the output of your compiler, please let us know. +goal are listed under _<>_. If you discover an undocumented case where +ccache changes the output of your compiler, please let us know. -Run modes ---------- +== Run modes There are two ways to use ccache. You can either prefix your compilation -commands with *ccache* or you can let ccache masquerade as the compiler by +commands with `ccache` or you can let ccache masquerade as the compiler by creating a symbolic link (named as the compiler) to ccache. The first method is most convenient if you just want to try out ccache or wish to use it for some specific projects. The second method is most useful for when you wish to use ccache for all your compilations. -To use the first method, just make sure that *ccache* is in your *PATH*. +To use the first method, just make sure that `ccache` is in your `PATH`. To use the symlinks method, do something like this: @@ -58,29 +50,27 @@ And so forth. This will work as long as the directory with symlinks comes before the path to the compiler (which is usually in `/usr/bin`). After -installing you may wish to run ``which gcc'' to make sure that the correct link +installing you may wish to run "`which gcc`" to make sure that the correct link is being used. WARNING: The technique of letting ccache masquerade as the compiler works well, -but currently doesn't interact well with other tools that do the same thing. -See _<<_using_ccache_with_other_compiler_wrappers,Using ccache with other -compiler wrappers>>_. +but currently doesn't interact well with other tools that do the same thing. See +_<>_. WARNING: Use a symbolic links for masquerading, not hard links. -Command line options --------------------- -These command line options only apply when you invoke ccache as ``ccache''. +== Command line options + +These command line options only apply when you invoke ccache as "`ccache`". When invoked as a compiler (via a symlink as described in the previous section), the normal compiler options apply and you should refer to the compiler's documentation. -Common options -~~~~~~~~~~~~~~ +=== Common options -*`-c`*, *`--cleanup`*:: +*-c*, *--cleanup*:: Clean up the cache by removing old cached files until the specified file number and cache size limits are not exceeded. This also recalculates the @@ -90,41 +80,47 @@ cleanup is mostly useful if you manually modify the cache contents or believe that the cache size statistics may be inaccurate. -*`-C`*, *`--clear`*:: +*-C*, *--clear*:: Clear the entire cache, removing all cached files, but keeping the configuration file. -*`--config-path`* _PATH_:: +*--config-path* _PATH_:: + + Let the command line options operate on configuration file _PATH_ instead of + the default. Using this option has the same effect as setting the + environment variable `CCACHE_CONFIGPATH` temporarily. + +*-d*, *--dir* _PATH_:: - Let the subsequent command line options operate on configuration file - _PATH_ instead of the default. Using this option has the same effect as - setting the environment variable `CCACHE_CONFIGPATH` temporarily. + Let the command line options operate on cache directory _PATH_ instead of + the default. For example, to show statistics for a cache directory at + `/shared/ccache` you can run `ccache -d /shared/ccache -s`. Using this option + has the same effect as setting the environment variable `CCACHE_DIR` + temporarily. -*`-d`*, *`--directory`* _PATH_:: +*--evict-namespace* _NAMESPACE_:: - Let the subsequent command line options operate on cache directory _PATH_ - instead of the default. For example, to show statistics for a cache - directory at `/shared/ccache` you can run `ccache -d /shared/ccache -s`. - Using this option has the same effect as setting the environment variable - `CCACHE_DIR` temporarily. + Remove files created in the given <> from the + cache. -*`--evict-older-than`* _AGE_:: +*--evict-older-than* _AGE_:: Remove files older than _AGE_ from the cache. _AGE_ should be an unsigned - integer with a `d` (days) or `s` (seconds) suffix. + integer with a `d` (days) or `s` (seconds) suffix. If combined with + `--evict-namespace`, only remove old files within that namespace. -*`-h`*, *`--help`*:: +*-h*, *--help*:: Print a summary of command line options. -*`-F`* _NUM_, *`--max-files`* _NUM_:: +*-F* _NUM_, *--max-files* _NUM_:: Set the maximum number of files allowed in the cache to _NUM_. Use 0 for no limit. The value is stored in a configuration file in the cache directory and applies to all future compilations. -*`-M`* _SIZE_, *`--max-size`* _SIZE_:: +*-M* _SIZE_, *--max-size* _SIZE_:: Set the maximum size of the files stored in the cache. _SIZE_ should be a number followed by an optional suffix: k, M, G, T (decimal), Ki, Mi, Gi or @@ -132,97 +128,136 @@ stored in a configuration file in the cache directory and applies to all future compilations. -*`-X`* _LEVEL_, *`--recompress`* _LEVEL_:: +*-X* _LEVEL_, *--recompress* _LEVEL_:: Recompress the cache to level _LEVEL_ using the Zstandard algorithm. The level can be an integer, with the same semantics as the <> configuration option), or the special value *uncompressed* for no compression. See - _<<_cache_compression,Cache compression>>_ for more information. This can - potentionally take a long time since all files in the cache need to be - visited. Only files that are currently compressed with a different level - than _LEVEL_ will be recompressed. + _<>_ for more information. This can potentionally take a + long time since all files in the cache need to be visited. Only files that + are currently compressed with a different level than _LEVEL_ will be + recompressed. -*`-o`* _KEY=VALUE_, *`--set-config`* _KEY_=_VALUE_:: +*-o* _KEY=VALUE_, *--set-config* _KEY_=_VALUE_:: - Set configuration option _KEY_ to _VALUE_. See - _<<_configuration,Configuration>>_ for more information. + Set configuration option _KEY_ to _VALUE_. See _<>_ for more + information. -*`-x`*, *`--show-compression`*:: +*-x*, *--show-compression*:: - Print cache compression statistics. See _<<_cache_compression,Cache - compression>>_ for more information. This can potentionally take a long - time since all files in the cache need to be visited. + Print cache compression statistics. See _<>_ for more + information. This can potentionally take a long time since all files in the + cache need to be visited. -*`-p`*, *`--show-config`*:: +*-p*, *--show-config*:: Print current configuration options and from where they originate (environment variable, configuration file or compile-time default) in human-readable format. -*`-s`*, *`--show-stats`*:: +*--show-log-stats*:: + + Print statistics counters from the stats log in human-readable format. See + <>. Use `-v`/`--verbose` once or twice for + more details. + +*-s*, *--show-stats*:: Print a summary of configuration and statistics counters in human-readable - format. + format. Use `-v`/`--verbose` once or twice for more details. + +*-v*, *--verbose*:: + + Increase verbosity. The option can be given multiple times. -*`-V`*, *`--version`*:: +*-V*, *--version*:: Print version and copyright information. -*`-z`*, *`--zero-stats`*:: +*-z*, *--zero-stats*:: Zero the cache statistics (but not the configuration options). -Options for scripting or debugging -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Options for secondary storage -*`--checksum-file`* _PATH_:: +*--trim-dir* _PATH_:: - Print the checksum (64 bit XXH3) of the file at _PATH_. + Remove old files from directory _PATH_ until it is at most the size specified + by `--trim-max-size`. ++ +WARNING: Don't use this option to trim the primary cache. To trim the primary +cache directory to a certain size, use `CCACHE_MAXSIZE=_SIZE_ ccache -c`. + +*--trim-max-size* _SIZE_:: + + Specify the maximum size for `--trim-dir`. _SIZE_ should be a number followed + by an optional suffix: k, M, G, T (decimal), Ki, Mi, Gi or Ti (binary). The + default suffix is G. + +*--trim-method* _METHOD_:: -*`--dump-manifest`* _PATH_:: + Specify the method to trim a directory with `--trim-dir`. Possible values + are: ++ +-- +*atime*:: + LRU (least recently used) using the file access timestamp. This is the + default. +*mtime*:: + LRU (least recently used) using the file modification timestamp. +-- - Dump manifest file at _PATH_ in text format to standard output. This is - only useful when debugging ccache and its behavior. +=== Options for scripting or debugging -*`--dump-result`* _PATH_:: +*--checksum-file* _PATH_:: - Dump result file at _PATH_ in text format to standard output. This is only - useful when debugging ccache and its behavior. + Print the checksum (128 bit XXH3) of the file at _PATH_ (`-` for standard + input). -*`--extract-result`* _PATH_:: +*--dump-manifest* _PATH_:: - Extract data stored in the result file at _PATH_. The data will be written - to *ccache-result.** files in to the current working directory. This is - only useful when debugging ccache and its behavior. + Dump manifest file at _PATH_ (`-` for standard input) in text format to + standard output. This is only useful when debugging ccache and its behavior. -*`-k`* _KEY_, *`--get-config`* _KEY_:: +*--dump-result* _PATH_:: - Print the value of configuration option _KEY_. See - _<<_configuration,Configuration>>_ for more information. + Dump result file at _PATH_ (`-` for standard input) in text format to + standard output. This is only useful when debugging ccache and its behavior. -*`--hash-file`* _PATH_:: +*--extract-result* _PATH_:: - Print the hash (160 bit BLAKE3) of the file at _PATH_. This is only useful - when debugging ccache and its behavior. + Extract data stored in the result file at _PATH_ (`-` for standard input). + The data will be written to `ccache-result.*` files in to the current + working directory. This is only useful when debugging ccache and its + behavior. -*`--print-stats`*:: +*-k* _KEY_, *--get-config* _KEY_:: + + Print the value of configuration option _KEY_. See _<>_ for + more information. + +*--hash-file* _PATH_:: + + Print the hash (160 bit BLAKE3) of the file at _PATH_ (`-` for standard + input). This is only useful when debugging ccache and its behavior. + +*--print-stats*:: Print statistics counter IDs and corresponding values in machine-parsable (tab-separated) format. -Extra options -~~~~~~~~~~~~~ +=== Extra options When run as a compiler, ccache usually just takes the same command line options as the compiler you are using. The only exception to this is the option -*--ccache-skip*. That option can be used to tell ccache to avoid interpreting +`--ccache-skip`. That option can be used to tell ccache to avoid interpreting the next option in any way and to pass it along to the compiler as-is. -NOTE: *--ccache-skip* currently only tells ccache not to interpret the next +NOTE: `--ccache-skip` currently only tells ccache not to interpret the next option as a special compiler option -- the option will still be included in the direct mode hash. @@ -231,20 +266,19 @@ it needs the input filename to determine the name of the resulting object file (among other things). The heuristic ccache uses when parsing the command line is that any argument that exists as a file is treated as an input file name. By -using *--ccache-skip* you can force an option to not be treated as an input +using `--ccache-skip` you can force an option to not be treated as an input file name and instead be passed along to the compiler as a command line option. -Another case where *--ccache-skip* can be useful is if ccache interprets an +Another case where `--ccache-skip` can be useful is if ccache interprets an option specially but shouldn't, since the option has another meaning for your compiler than what ccache thinks. -Configuration -------------- +== Configuration -ccache's default behavior can be overridden by options in configuration files, +Ccache's default behavior can be overridden by options in configuration files, which in turn can be overridden by environment variables with names starting -with *CCACHE_*. Ccache normally reads configuration from two files: first a +with `CCACHE_`. Ccache normally reads configuration from two files: first a system-level configuration file and secondly a cache-specific configuration file. The priorities of configuration options are as follows (where 1 is highest): @@ -252,39 +286,37 @@ 1. Environment variables. 2. The primary (cache-specific) configuration file (see below). 3. The secondary (system-wide read-only) configuration file - *__/ccache.conf* (typically */etc/ccache.conf* or - */usr/local/etc/ccache.conf*). + `/ccache.conf` (typically `/etc/ccache.conf` or + `/usr/local/etc/ccache.conf`). 4. Compile-time defaults. -As a special case, if the the environment variable *CCACHE_CONFIGPATH* is set +As a special case, if the the environment variable `CCACHE_CONFIGPATH` is set it specifies the primary configuration file and the secondary (system-wide) configuration file won't be read. -Location of the primary configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Location of the primary configuration file The location of the primary (cache-specific) configuration is determined like this: -1. If *CCACHE_CONFIGPATH* is set, use that path. -2. Otherwise, if the environment variable *CCACHE_DIR* is set then use - *$CCACHE_DIR/ccache.conf*. +1. If `CCACHE_CONFIGPATH` is set, use that path. +2. Otherwise, if the environment variable `CCACHE_DIR` is set then use + `$CCACHE_DIR/ccache.conf`. 3. Otherwise, if <> is set in the secondary - (system-wide) configuration file then use */ccache.conf*. -4. Otherwise, if there is a legacy *$HOME/.ccache* directory then use - *$HOME/.ccache/ccache.conf*. -5. Otherwise, if *XDG_CONFIG_HOME* is set then use - *$XDG_CONFIG_HOME/ccache/ccache.conf*. -6. Otherwise, use *%APPDATA%/ccache/ccache.conf* (Windows), - *$HOME/Library/Preferences/ccache/ccache.conf* (macOS) or - *$HOME/.config/ccache/ccache.conf* (other systems). + (system-wide) configuration file then use `/ccache.conf`. +4. Otherwise, if there is a legacy `$HOME/.ccache` directory then use + `$HOME/.ccache/ccache.conf`. +5. Otherwise, if `XDG_CONFIG_HOME` is set then use + `$XDG_CONFIG_HOME/ccache/ccache.conf`. +6. Otherwise, use `%APPDATA%/ccache/ccache.conf` (Windows), + `$HOME/Library/Preferences/ccache/ccache.conf` (macOS) or + `$HOME/.config/ccache/ccache.conf` (other systems). -Configuration file syntax -~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Configuration file syntax -Configuration files are in a simple ``key = value'' format, one option per +Configuration files are in a simple "`key = value`" format, one option per line. Lines starting with a hash sign are comments. Blank lines are ignored, as is whitespace surrounding keys and values. Example: @@ -293,31 +325,31 @@ max_size = 10G ------------------------------------------------------------------------------- -Boolean values -~~~~~~~~~~~~~~ + +=== Boolean values Some configuration options are boolean values (i.e. truth values). In a configuration file, such values must be set to the string *true* or *false*. For the corresponding environment variables, the semantics are a bit different: -* A set environment variable means ``true'' (even if set to the empty string). +* A set environment variable means "`true`" (even if set to the empty string). * The following case-insensitive negative values are considered an error (instead of surprising the user): *0*, *false*, *disable* and *no*. -* An unset environment variable means ``false''. +* An unset environment variable means "`false`". Each boolean environment variable also has a negated form starting with -*CCACHE_NO*. For example, *CCACHE_COMPRESS* can be set to force compression and -*CCACHE_NOCOMPRESS* can be set to force no compression. +`CCACHE_NO`. For example, `CCACHE_COMPRESS` can be set to force compression and +`CCACHE_NOCOMPRESS` can be set to force no compression. -Configuration options -~~~~~~~~~~~~~~~~~~~~~~ +=== Configuration options Below is a list of available configuration options. The corresponding environment variable name is indicated in parentheses after each configuration option key. -[[config_absolute_paths_in_stderr]] *absolute_paths_in_stderr* (*CCACHE_ABSSTDERR*):: +[#config_absolute_paths_in_stderr] +*absolute_paths_in_stderr* (*CCACHE_ABSSTDERR*):: This option specifies whether ccache should rewrite relative paths in the compiler's standard error output to absolute paths. This can be useful if @@ -326,16 +358,16 @@ working directory, which makes relative paths in compiler errors or warnings incorrect. The default is false. -[[config_base_dir]] *base_dir* (*CCACHE_BASEDIR*):: +[#config_base_dir] +*base_dir* (*CCACHE_BASEDIR*):: This option should be an absolute path to a directory. If set, ccache will - rewrite absolute paths into paths relative to the current working - directory, but only absolute paths that begin with *base_dir*. Cache - results can then be shared for compilations in different directories even - if the project uses absolute paths in the compiler command line. See also - the discussion under _<<_compiling_in_different_directories,Compiling in - different directories>>_. If set to the empty string (which is the - default), no rewriting is done. + rewrite absolute paths into paths relative to the current working directory, + but only absolute paths that begin with *base_dir*. Cache results can then + be shared for compilations in different directories even if the project uses + absolute paths in the compiler command line. See also the discussion under + _<>_. If set to the empty string (which + is the default), no rewriting is done. + A typical path to use as *base_dir* is your home directory or another directory that is a parent of your project directories. Don't use `/` as the base @@ -382,28 +414,30 @@ to `/home/bob/stuff/project1` there will a cache miss since the path to project2 will be a different absolute path. -[[config_cache_dir]] *cache_dir* (*CCACHE_DIR*):: +[#config_cache_dir] +*cache_dir* (*CCACHE_DIR*):: This option specifies where ccache will keep its cached compiler outputs. - The default is *$XDG_CACHE_HOME/ccache* if *XDG_CACHE_HOME* is set, - otherwise *$HOME/.cache/ccache*. Exception: If the legacy directory - *$HOME/.ccache* exists then that directory is the default. + The default is `$XDG_CACHE_HOME/ccache` if `XDG_CACHE_HOME` is set, + otherwise `$HOME/.cache/ccache`. Exception: If the legacy directory + `$HOME/.ccache` exists then that directory is the default. + -See also _<<_location_of_the_primary_configuration_file,Location of the primary -configuration file>>_. +See also _<>_. + -If you want to use another *CCACHE_DIR* value temporarily for one ccache -invocation you can use the `-d/--directory` command line option instead. +If you want to use another `CCACHE_DIR` value temporarily for one ccache +invocation you can use the `-d`/`--dir` command line option instead. -[[config_compiler]] *compiler* (*CCACHE_COMPILER* or (deprecated) *CCACHE_CC*):: +[#config_compiler] +*compiler* (*CCACHE_COMPILER* or (deprecated) *CCACHE_CC*):: This option can be used to force the name of the compiler to use. If set to the empty string (which is the default), ccache works it out from the command line. -[[config_compiler_check]] *compiler_check* (*CCACHE_COMPILERCHECK*):: +[#config_compiler_check] +*compiler_check* (*CCACHE_COMPILERCHECK*):: - By default, ccache includes the modification time (``mtime'') and size of + By default, ccache includes the modification time ("`mtime`") and size of the compiler in the hash to ensure that results retrieved from the cache are accurate. This option can be used to select another strategy. Possible values are: @@ -433,35 +467,32 @@ path to the compiler. Several commands can be specified with semicolon as separator. Examples: + --- - ---- %compiler% -v ---- - ++ ---- %compiler% -dumpmachine; %compiler% -dumpversion ---- - ++ You should make sure that the specified command is as fast as possible since it will be run once for each ccache invocation. - ++ Identifying the compiler using a command is useful if you want to avoid cache misses when the compiler has been rebuilt but not changed. - ++ Another case is when the compiler (as seen by ccache) actually isn't the real compiler but another compiler wrapper -- in that case, the default *mtime* method will hash the mtime and size of the other compiler wrapper, which means -that ccache won't be able to detect a compiler upgrade. Using a suitable -command to identify the compiler is thus safer, but it's also slower, so you -should consider continue using the *mtime* method in combination with the +that ccache won't be able to detect a compiler upgrade. Using a suitable command +to identify the compiler is thus safer, but it's also slower, so you should +consider continue using the *mtime* method in combination with the *prefix_command* option if possible. See -_<<_using_ccache_with_other_compiler_wrappers,Using ccache with other compiler -wrappers>>_. --- +_<>_. -- -[[config_compiler_type]] *compiler_type* (*CCACHE_COMPILERTYPE*):: +[#config_compiler_type] +*compiler_type* (*CCACHE_COMPILERTYPE*):: Ccache normally guesses the compiler type based on the compiler name. The *compiler_type* option lets you force a compiler type. This can be useful @@ -481,10 +512,11 @@ *other*:: Any compiler other than the known types. *pump*:: - distcc's "pump" script. + distcc's "`pump`" script. -- -[[config_compression]] *compression* (*CCACHE_COMPRESS* or *CCACHE_NOCOMPRESS*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_compression] +*compression* (*CCACHE_COMPRESS* or *CCACHE_NOCOMPRESS*, see _<>_ above):: If true, ccache will compress data it puts in the cache. However, this option has no effect on how files are retrieved from the cache; compressed @@ -500,7 +532,8 @@ <> option) or hard linking (the <> option) is enabled. -[[config_compression_level]] *compression_level* (*CCACHE_COMPRESSLEVEL*):: +[#config_compression_level] +*compression_level* (*CCACHE_COMPRESSLEVEL*):: This option determines the level at which ccache will compress object files using the real-time compression algorithm Zstandard. It only has effect if @@ -520,11 +553,11 @@ essentially the same for all levels. As a rule of thumb, use level 5 or lower since higher levels may slow down compilations noticeably. Higher levels are however useful when recompressing the cache with command line - option *-X/--recompress*. + option `-X`/`--recompress`. *< 0*:: - A negative value corresponds to Zstandard's ``ultra-fast'' compression + A negative value corresponds to Zstandard's "`ultra-fast`" compression levels, which are even faster than level 1 but with less good compression - ratios. For instance, level *-3* corresponds to ``--fast=3'' for the *zstd* + ratios. For instance, level *-3* corresponds to `--fast=3` for the `zstd` command line tool. In practice, there is little use for levels lower than *-5* or so. *0* (default):: @@ -534,26 +567,28 @@ + See the http://zstd.net[Zstandard documentation] for more information. -[[config_cpp_extension]] *cpp_extension* (*CCACHE_EXTENSION*):: +[#config_cpp_extension] +*cpp_extension* (*CCACHE_EXTENSION*):: This option can be used to force a certain extension for the intermediate preprocessed file. The default is to automatically determine the extension to use for intermediate preprocessor files based on the type of file being compiled, but that sometimes doesn't work. For example, when using the - ``aCC'' compiler on HP-UX, set the cpp extension to *i*. + "`aCC`" compiler on HP-UX, set the cpp extension to *i*. -[[config_debug]] *debug* (*CCACHE_DEBUG* or *CCACHE_NODEBUG*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_debug] +*debug* (*CCACHE_DEBUG* or *CCACHE_NODEBUG*, see _<>_ above):: If true, enable the debug mode. The debug mode creates per-object debug files that are helpful when debugging unexpected cache misses. Note however - that ccache performance will be reduced slightly. See - _<<_cache_debugging,Cache debugging>>_ for more information. The default is - false. + that ccache performance will be reduced slightly. See _<>_ + for more information. The default is false. -[[config_debug_dir]] *debug_dir* (*CCACHE_DEBUGDIR*):: +[#config_debug_dir] +*debug_dir* (*CCACHE_DEBUGDIR*):: - Specifies where to write per-object debug files if the _<>_ is enabled. If set to the empty string, the files will be written + Specifies where to write per-object debug files if the <> is enabled. If set to the empty string, the files will be written next to the object file. If set to a directory, the debug files will be written with full absolute paths in that directory, creating it if needed. The default is the empty string. @@ -561,35 +596,40 @@ For example, if *debug_dir* is set to `/example`, the current working directory is `/home/user` and the object file is `build/output.o` then the debug log will be written to `/example/home/user/build/output.o.ccache-log`. See also -_<<_cache_debugging,Cache debugging>>_. +_<>_. -[[config_depend_mode]] *depend_mode* (*CCACHE_DEPEND* or *CCACHE_NODEPEND*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_depend_mode] +*depend_mode* (*CCACHE_DEPEND* or *CCACHE_NODEPEND*, see _<>_ above):: If true, the depend mode will be used. The default is false. See - _<<_the_depend_mode,The depend mode>>_. + _<>_. -[[config_direct_mode]] *direct_mode* (*CCACHE_DIRECT* or *CCACHE_NODIRECT*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_direct_mode] +*direct_mode* (*CCACHE_DIRECT* or *CCACHE_NODIRECT*, see _<>_ above):: If true, the direct mode will be used. The default is true. See - _<<_the_direct_mode,The direct mode>>_. + _<>_. -[[config_disable]] *disable* (*CCACHE_DISABLE* or *CCACHE_NODISABLE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_disable] +*disable* (*CCACHE_DISABLE* or *CCACHE_NODISABLE*, see _<>_ above):: When true, ccache will just call the real compiler, bypassing the cache completely. The default is false. -[[config_extra_files_to_hash]] *extra_files_to_hash* (*CCACHE_EXTRAFILES*):: +[#config_extra_files_to_hash] +*extra_files_to_hash* (*CCACHE_EXTRAFILES*):: This option is a list of paths to files that ccache will include in the the hash sum that identifies the build. The list separator is semicolon on Windows systems and colon on other systems. -[[config_file_clone]] *file_clone* (*CCACHE_FILECLONE* or *CCACHE_NOFILECLONE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_file_clone] +*file_clone* (*CCACHE_FILECLONE* or *CCACHE_NOFILECLONE*, see _<>_ above):: - If true, ccache will attempt to use file cloning (also known as ``copy on - write'', ``CoW'' or ``reflinks'') to store and fetch cached compiler results. - *file_clone* has priority over <>. The - default is false. + If true, ccache will attempt to use file cloning (also known as "`copy on + write`", "`CoW`" or "`reflinks`") to store and fetch cached compiler + results. *file_clone* has priority over <>. + The default is false. + Files stored by cloning cannot be compressed, so the cache size will likely be significantly larger if this option is enabled. However, performance may be @@ -600,7 +640,8 @@ systems, ccache will fall back to use plain copying (or hard links if <> is enabled). -[[config_hard_link]] *hard_link* (*CCACHE_HARDLINK* or *CCACHE_NOHARDLINK*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_hard_link] +*hard_link* (*CCACHE_HARDLINK* or *CCACHE_NOHARDLINK*, see _<>_ above):: If true, ccache will attempt to use hard links to store and fetch cached object files. The default is false. @@ -621,25 +662,25 @@ file size will not. * Programs that don't expect that files from two different identical compilations are hard links to each other can fail. -* Programs that rely on modification times (like ``make'') can be confused if +* Programs that rely on modification times (like `make`) can be confused if several users (or one user with several build trees) use the same cache directory. The reason for this is that the object files share i-nodes and - therefore modification times. If *file.o* is in build tree A (hard-linked - from the cache) and *file.o* then is produced by ccache in build tree B by + therefore modification times. If `file.o` is in build tree *A* (hard-linked + from the cache) and `file.o` then is produced by ccache in build tree *B* by hard-linking from the cache, the modification timestamp will be updated for - *file.o* in build tree A as well. This can retrigger relinking in build tree - A even though nothing really has changed. + `file.o` in build tree *A* as well. This can retrigger relinking in build tree + *A* even though nothing really has changed. -[[config_hash_dir]] *hash_dir* (*CCACHE_HASHDIR* or *CCACHE_NOHASHDIR*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_hash_dir] +*hash_dir* (*CCACHE_HASHDIR* or *CCACHE_NOHASHDIR*, see _<>_ above):: If true (which is the default), ccache will include the current working directory (CWD) in the hash that is used to distinguish two compilations - when generating debug info (compiler option *-g* with variations). + when generating debug info (compiler option `-g` with variations). Exception: The CWD will not be included in the hash if <> is set (and matches the CWD) and the - compiler option *-fdebug-prefix-map* is used. See also the discussion under - _<<_compiling_in_different_directories,Compiling in different - directories>>_. + compiler option `-fdebug-prefix-map` is used. See also the discussion under + _<>_. + The reason for including the CWD in the hash by default is to prevent a problem with the storage of the current working directory in the debug info of an @@ -650,7 +691,8 @@ code in different directories if you don't mind that CWD in the debug info might be incorrect. -[[config_ignore_headers_in_manifest]] *ignore_headers_in_manifest* (*CCACHE_IGNOREHEADERS*):: +[#config_ignore_headers_in_manifest] +*ignore_headers_in_manifest* (*CCACHE_IGNOREHEADERS*):: This option is a list of paths to files (or directories with headers) that ccache will *not* include in the manifest list that makes up the direct @@ -658,17 +700,19 @@ change. The list separator is semicolon on Windows systems and colon on other systems. -[[config_ignore_options]] *ignore_options* (*CCACHE_IGNOREOPTIONS*):: +[#config_ignore_options] +*ignore_options* (*CCACHE_IGNOREOPTIONS*):: This option is a space-delimited list of compiler options that ccache will exclude from the hash. Excluding a compiler option from the hash can be useful when you know it doesn't affect the result (but ccache doesn't know that), or when it does and you don't care. If a compiler option in the list is suffixed with an asterisk (`*`) it will be matched as a prefix. For - example, `-fmessage-length=*` will match both `-fmessage-length=20` and + example, `+-fmessage-length=*+` will match both `-fmessage-length=20` and `-fmessage-length=70`. -[[config_inode_cache]] *inode_cache* (*CCACHE_INODECACHE* or *CCACHE_NOINODECACHE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_inode_cache] +*inode_cache* (*CCACHE_INODECACHE* or *CCACHE_NOINODECACHE*, see _<>_ above):: If true, enables caching of source file hashes based on device, inode and timestamps. This will reduce the time spent on hashing included files as @@ -679,18 +723,21 @@ + The feature requires *temporary_dir* to be located on a local filesystem. -[[config_keep_comments_cpp]] *keep_comments_cpp* (*CCACHE_COMMENTS* or *CCACHE_NOCOMMENTS*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_keep_comments_cpp] +*keep_comments_cpp* (*CCACHE_COMMENTS* or *CCACHE_NOCOMMENTS*, see _<>_ above):: If true, ccache will not discard the comments before hashing preprocessor - output. This can be used to check documentation with *-Wdocumentation*. + output. This can be used to check documentation with `-Wdocumentation`. -[[config_limit_multiple]] *limit_multiple* (*CCACHE_LIMIT_MULTIPLE*):: +[#config_limit_multiple] +*limit_multiple* (*CCACHE_LIMIT_MULTIPLE*):: Sets the limit when cleaning up. Files are deleted (in LRU order) until the levels are below the limit. The default is 0.8 (= 80%). See - _<<_automatic_cleanup,Automatic cleanup>>_ for more information. + _<>_ for more information. -[[config_log_file]] *log_file* (*CCACHE_LOGFILE*):: +[#config_log_file] +*log_file* (*CCACHE_LOGFILE*):: If set to a file path, ccache will write information on what it is doing to the specified file. This is useful for tracking down problems. @@ -706,76 +753,106 @@ & ~ ------------------------------------------------------------------------------- -[[config_max_files]] *max_files* (*CCACHE_MAXFILES*):: +[#config_max_files] +*max_files* (*CCACHE_MAXFILES*):: This option specifies the maximum number of files to keep in the cache. Use - 0 for no limit (which is the default). See also - _<<_cache_size_management,Cache size management>>_. + 0 for no limit (which is the default). See also _<>_. -[[config_max_size]] *max_size* (*CCACHE_MAXSIZE*):: +[#config_max_size] +*max_size* (*CCACHE_MAXSIZE*):: - This option specifies the maximum size of the cache. Use 0 for no limit. - The default value is 5G. Available suffixes: k, M, G, T (decimal) and Ki, - Mi, Gi, Ti (binary). The default suffix is G. See also - _<<_cache_size_management,Cache size management>>_. + This option specifies the maximum size of the cache. Use 0 for no limit. The + default value is 5G. Available suffixes: k, M, G, T (decimal) and Ki, Mi, + Gi, Ti (binary). The default suffix is G. See also + _<>_. + +[#config_namespace] +*namespace* (*CCACHE_NAMESPACE*):: + + If set, the namespace string will be added to the hashed data for each + compilation. This will make the associated cache entries logically separate + from cache entries with other namespaces, but they will still share the same + storage space. Cache entries can also be selectively removed from the + primary cache with the command line option `--evict-namespace`, potentially + in combination with `--evict-older-than`. +. +For instance, if you use the same primary cache for several disparate projects, +you can use a unique namespace string for each one. This allows you to remove +cache entries that belong to a certain project if stop working with that +project. -[[config_path]] *path* (*CCACHE_PATH*):: +[#config_path] +*path* (*CCACHE_PATH*):: If set, ccache will search directories in this list when looking for the real compiler. The list separator is semicolon on Windows systems and colon on other systems. If not set, ccache will look for the first executable - matching the compiler name in the normal *PATH* that isn't a symbolic link + matching the compiler name in the normal `PATH` that isn't a symbolic link to ccache itself. -[[config_pch_external_checksum]] *pch_external_checksum* (*CCACHE_PCH_EXTSUM* or *CCACHE_NOPCH_EXTSUM*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_pch_external_checksum] +*pch_external_checksum* (*CCACHE_PCH_EXTSUM* or *CCACHE_NOPCH_EXTSUM*, see _<>_ above):: When this option is set, and ccache finds a precompiled header file, - ccache will look for a file with the extension ``.sum'' added - (e.g. ``pre.h.gch.sum''), and if found, it will hash this file instead + ccache will look for a file with the extension "`.sum`" added + (e.g. "`pre.h.gch.sum`"), and if found, it will hash this file instead of the precompiled header itself to work around the performance penalty of hashing very large files. -[[config_prefix_command]] *prefix_command* (*CCACHE_PREFIX*):: +[#config_prefix_command] +*prefix_command* (*CCACHE_PREFIX*):: - This option adds a list of prefixes (separated by space) to the command - line that ccache uses when invoking the compiler. See also - _<<_using_ccache_with_other_compiler_wrappers,Using ccache with other - compiler wrappers>>_. + This option adds a list of prefixes (separated by space) to the command line + that ccache uses when invoking the compiler. See also + _<>_. -[[config_prefix_command_cpp]] *prefix_command_cpp* (*CCACHE_PREFIX_CPP*):: +[#config_prefix_command_cpp] +*prefix_command_cpp* (*CCACHE_PREFIX_CPP*):: This option adds a list of prefixes (separated by space) to the command line that ccache uses when invoking the preprocessor. -[[config_read_only]] *read_only* (*CCACHE_READONLY* or *CCACHE_NOREADONLY*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_read_only] +*read_only* (*CCACHE_READONLY* or *CCACHE_NOREADONLY*, see _<>_ above):: If true, ccache will attempt to use existing cached results, but it will not - add new results to the cache. Statistics counters will still be updated, - though, unless the <> option is set to *false*. + add new results to any cache backend. Statistics counters will still be + updated, though, unless the <> option is set to + *false*. + If you are using this because your ccache directory is read-only, you need to set <> since ccache will fail to create temporary files otherwise. You may also want to set <> to *false* make ccache not even try to update stats files. -[[config_read_only_direct]] *read_only_direct* (*CCACHE_READONLY_DIRECT* or *CCACHE_NOREADONLY_DIRECT*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_read_only_direct] +*read_only_direct* (*CCACHE_READONLY_DIRECT* or *CCACHE_NOREADONLY_DIRECT*, see _<>_ above):: Just like <> except that ccache will only try to retrieve results from the cache using the direct mode, not the preprocessor mode. See documentation for <> regarding using a read-only ccache directory. -[[config_recache]] *recache* (*CCACHE_RECACHE* or *CCACHE_NORECACHE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_recache] +*recache* (*CCACHE_RECACHE* or *CCACHE_NORECACHE*, see _<>_ above):: If true, ccache will not use any previously stored result. New results will still be cached, possibly overwriting any pre-existing results. -[[config_run_second_cpp]] *run_second_cpp* (*CCACHE_CPP2* or *CCACHE_NOCPP2*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_reshare] +*reshare* (*CCACHE_RESHARE* or *CCACHE_NORESHARE*, see _<>_ above):: + + If true, ccache will write results to secondary storage even for primary + storage cache hits. The default is false. + +[#config_run_second_cpp] +*run_second_cpp* (*CCACHE_CPP2* or *CCACHE_NOCPP2*, see _<>_ above):: If true, ccache will first run the preprocessor to preprocess the source - code (see _<<_the_preprocessor_mode,The preprocessor mode>>_) and then on a - cache miss run the compiler on the source code to get hold of the object - file. This is the default. + code (see _<>_) and then on a cache miss run the + compiler on the source code to get hold of the object file. This is the + default. + If false, ccache will first run preprocessor to preprocess the source code and then on a cache miss run the compiler on the _preprocessed source code_ instead @@ -785,13 +862,29 @@ when compiling preprocessed source code. + A solution to the above mentioned downside is to set *run_second_cpp* to false -and pass *-fdirectives-only* (for GCC) or *-frewrite-includes* (for Clang) to +and pass `-fdirectives-only` (for GCC) or `-frewrite-includes` (for Clang) to the compiler. This will cause the compiler to leave the macros and other preprocessor information, and only process the *#include* directives. When run in this way, the preprocessor arguments will be passed to the compiler since it still has to do _some_ preprocessing (like macros). -[[config_sloppiness]] *sloppiness* (*CCACHE_SLOPPINESS*):: +[#config_secondary_storage] +*secondary_storage* (*CCACHE_SECONDARY_STORAGE*):: + + This option specifies one or several storage backends (separated by space) + to query after the primary cache storage. See + _<>_ for documentation of syntax and available + backends. ++ +Examples: ++ +* `+file:/shared/nfs/directory+` +* `+file:///shared/nfs/one|read-only file:///shared/nfs/two+` +* `+http://example.com/cache+` +* `+redis://example.com+` + +[#config_sloppiness] +*sloppiness* (*CCACHE_SLOPPINESS*):: By default, ccache tries to give as few false cache hits as possible. However, in certain situations it's possible that you know things that @@ -802,7 +895,7 @@ + -- *clang_index_store*:: - Ignore the Clang compiler option *-index-store-path* and its argument when + Ignore the Clang compiler option `-index-store-path` and its argument when computing the manifest hash. This is useful if you use Xcode, which uses an index store path derived from the local project path. Note that the index store won't be updated correctly on cache hits if you enable this @@ -817,73 +910,266 @@ *include_file_ctime*:: By default, ccache will not cache a file if it includes a header whose ctime is too new. This sloppiness disables that check. See also - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. + _<>_. *include_file_mtime*:: - By default, ccache will not cache a file if it includes a header whose - mtime is too new. This sloppiness disables that check. See also - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. + By default, ccache will not cache a file if it includes a header whose mtime + is too new. This sloppiness disables that check. See also + _<>_. +*ivfsoverlay*:: + Ignore the Clang compiler option `-ivfsoverlay` and its argument. This is + useful if you use Xcode, which uses a virtual file system (VFS) for things + like combining Objective-C and Swift code. *locale*:: - Ccache includes the environment variables *LANG*, *LC_ALL*, *LC_CTYPE* and - *LC_MESSAGES* in the hash by default since they may affect localization of + Ccache includes the environment variables `LANG`, `LC_ALL`, `LC_CTYPE` and + `LC_MESSAGES` in the hash by default since they may affect localization of compiler warning messages. Set this sloppiness to tell ccache not to do that. *pch_defines*:: - Be sloppy about **#define**s when precompiling a header file. See - _<<_precompiled_headers,Precompiled headers>>_ for more information. + Be sloppy about `#define` directives when precompiling a header file. See + _<>_ for more information. *modules*:: - By default, ccache will not cache compilations if *-fmodules* is used since + By default, ccache will not cache compilations if `-fmodules` is used since it cannot hash the state of compiler's internal representation of relevant modules. This sloppiness allows caching in such a case. See - _<<_c_modules,C++ modules>>_ for more information. + _<>_ for more information. *system_headers*:: By default, ccache will also include all system headers in the manifest. With this sloppiness set, ccache will only include system headers in the hash but not add the system header files to the list of include files. *time_macros*:: - Ignore `__DATE__`, `__TIME__` and `__TIMESTAMP__` being present in the + Ignore `+__DATE__+`, `+__TIME__+` and `+__TIMESTAMP__+` being present in the source code. -- + -See the discussion under _<<_troubleshooting,Troubleshooting>>_ for more -information. +See the discussion under _<>_ for more information. -[[config_stats]] *stats* (*CCACHE_STATS* or *CCACHE_NOSTATS*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_stats] +*stats* (*CCACHE_STATS* or *CCACHE_NOSTATS*, see _<>_ above):: If true, ccache will update the statistics counters on each compilation. The default is true. -[[config_temporary_dir]] *temporary_dir* (*CCACHE_TEMPDIR*):: +[#config_stats_log] +*stats_log* (*CCACHE_STATSLOG*):: + + If set to a file path, ccache will write statistics counter updates to the + specified file. This is useful for getting statistics for individual builds. + To show a summary of the current stats log, use `ccache --show-log-stats`. ++ +NOTE: Lines in the stats log starting with a hash sign (`#`) are comments. + +[#config_temporary_dir] +*temporary_dir* (*CCACHE_TEMPDIR*):: This option specifies where ccache will put temporary files. The default is - */run/user//ccache-tmp* if */run/user/* exists, otherwise - */tmp*. + `/run/user//ccache-tmp` if `/run/user/` exists, otherwise + `/tmp`. + NOTE: In previous versions of ccache, *CCACHE_TEMPDIR* had to be on the same -filesystem as the *CCACHE_DIR* path, but this requirement has been relaxed.) +filesystem as the `CCACHE_DIR` path, but this requirement has been relaxed. + +[#config_umask] +*umask* (*CCACHE_UMASK*):: + + This option (an octal integer) specifies the umask for files and directories + in the cache directory. This is mostly useful when you wish to share your + cache with other users. + + +== Secondary storage backends + +The <> option lets you configure +ccache to use one or several other storage backends in addition to the primary +cache storage located in <>. Note that cache +statistics counters will still be kept in the primary cache directory -- +secondary storage backends only store cache results and manifests. + +A secondary storage backend is specified with a URL, optionally followed by a +pipe (`|`) and a pipe-separated list of attributes. An attribute is +_key_=_value_ or just _key_ as a short form of _key_=*true*. Attribute values +must be https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they +contain percent, pipe or space characters. + +=== Attributes for all backends + +These optional attributes are available for all secondary storage backends: + +* *read-only*: If *true*, only read from this backend, don't write. The default + is *false*. +* *shards*: A comma-separated list of names for sharding (partitioning) the + cache entries using + https://en.wikipedia.org/wiki/Rendezvous_hashing[Rendezvous hashing], + typically to spread the cache over a server cluster. When set, the storage URL + must contain an asterisk (`+*+`), which will be replaced by one of the shard + names to form a real URL. A shard name can optionally have an appended weight + within parentheses to indicate how much of the key space should be associated + with that shard. A shard with weight *w* will contain *w*/*S* of the cache, + where *S* is the sum of all shard weights. A weight could for instance be set + to represent the available memory for a memory cache on a specific server. The + default weight is *1*. ++ +Examples: ++ +-- +* `+redis://cache-*.example.com|shards=a(3),b(1),c(1.5)+` will put 55% (3/5.5) + of the cache on `+redis://cache-a.example.com+`, 18% (1/5.5) on + `+redis://cache-b.example.com+` and 27% (1.5/5.5) on + `+redis://cache-c.example.com+`. +* `+http://example.com/*|shards=alpha,beta+` will put 50% of the cache on + `+http://example.com/alpha+` and 50% on `+http://example.com/beta+`. +-- +* *share-hits*: If *true*, write hits for this backend to primary storage. The + default is *true*. + + +=== Storage interaction + +The table below describes the interaction between primary and secondary storage +on cache hits and misses: + +[options="header",cols="20%,20%,60%"] +|============================================================================== +| *Primary storage* | *Secondary storage* | *What happens* + +| miss | miss | Compile, write to primary, write to secondary^[1]^ +| miss | hit | Read from secondary, write to primary^[2]^ +| hit | - | Read from primary, don't write to secondary^[3]^ + +|============================================================================== + +^[1]^ Unless secondary storage has attribute `read-only=true`. + +^[2]^ Unless secondary storage has attribute `share-hits=false`. + +^[3]^ Unless primary storage is set to share its cache hits with the +<> option. + + + +=== File storage backend + +URL format: `+file:DIRECTORY+` or `+file://DIRECTORY+` + +This backend stores data as separate files in a directory structure below +*DIRECTORY* (an absolute path), similar (but not identical) to the primary cache +storage. A typical use case for this backend would be sharing a cache on an NFS +directory. + +IMPORTANT: ccache will not perform any cleanup of the storage -- that has to be +done by other means, for instance by running `ccache --trim-dir` periodically. + +Examples: + +* `+file:/shared/nfs/directory+` +* `+file:///shared/nfs/directory|umask=002|update-mtime=true+` -[[config_umask]] *umask* (*CCACHE_UMASK*):: +Optional attributes: - This option specifies the umask for files and directories in the cache - directory. This is mostly useful when you wish to share your cache with - other users. +* *layout*: How to store file under the cache directory. Available values: ++ +-- +* *flat*: Store all files directly under the cache directory. +* *subdirs*: Store files in 256 subdirectories of the cache directory. +-- ++ +The default is *subdirs*. +* *umask*: This attribute (an octal integer) overrides the umask to use for + files and directories in the cache directory. +* *update-mtime*: If *true*, update the modification time (mtime) of cache + entries that are read. The default is *false*. + + +=== HTTP storage backend + +URL format: `+http://HOST[:PORT][/PATH]+` + +This backend stores data in an HTTP-compatible server. The required HTTP methods +are `GET`, `PUT` and `DELETE`. + +IMPORTANT: ccache will not perform any cleanup of the storage -- that has to be +done by other means, for instance by running `ccache --trim-dir` periodically. +NOTE: HTTPS is not supported. -Cache size management ---------------------- +TIP: See https://ccache.dev/howto/http-storage.html[How to set up HTTP storage] +for hints on how to set up an HTTP server for use with ccache. + +Examples: + +* `+http://localhost+` +* `+http://someusername:p4ssw0rd@example.com/cache/+` +* `+http://localhost:8080|layout=bazel|connect-timeout=50+` + +Optional attributes: + +* *connect-timeout*: Timeout (in ms) for network connection. The default is 100. +* *keep-alive*: If *true*, keep the HTTP connection to the storage server open + to avoid reconnects. The default is *false*. ++ +NOTE: Connection keep-alive is disabled by default because with the current +HTTP implementation uploads to the remote end might fail in case the server +closes the connection due to a keep-alive timeout. If the general case with +short compilation times should be accelerated or the server is configured with +a long-enough timeout, then connection keep-alive could be enabled. +* *layout*: How to map key names to the path part of the URL. Available values: ++ +-- +* *bazel*: Store values in a format compatible with the Bazel HTTP caching + protocol. More specifically, the entries will be stored as 64 hex digits + under the `/ac/` part of the cache. ++ +NOTE: You may have to disable verification of action cache values in the server +for this to work since ccache entries are not valid action result metadata +values. +* *flat*: Append the key directly to the path part of the URL (with a leading + slash if needed). +* *subdirs*: Append the first two characters of the key to the URL (with a + leading slash if needed), followed by a slash and the rest of the key. This + divides the entries into 256 buckets. +-- ++ +The default is *subdirs*. +* *operation-timeout*: Timeout (in ms) for HTTP requests. The default is 10000. + + +=== Redis storage backend + +URL format: `+redis://[[USERNAME:]PASSWORD@]HOST[:PORT][/DBNUMBER]+` + +This backend stores data in a https://redis.io[Redis] (or Redis-compatible) +server. There are implementations for both memory-based and disk-based storage. +*PORT* defaults to *6379* and *DBNUMBER* defaults to *0*. + +NOTE: ccache will not perform any cleanup of the Redis storage, but you can +https://redis.io/topics/lru-cache[configure LRU eviction]. + +TIP: See https://ccache.dev/howto/redis-storage.html[How to set up Redis +storage] for hints on setting up a Redis server for use with ccache. + +TIP: You can set up a cluster of Redis servers using the `shards` attribute +described in _<>_. + +Examples: + +* `+redis://localhost+` +* `+redis://p4ssw0rd@cache.example.com:6379/0|connect-timeout=50+` + +Optional attributes: + +* *connect-timeout*: Timeout (in ms) for network connection. The default is 100. +* *operation-timeout*: Timeout (in ms) for Redis commands. The default is 10000. + + +== Cache size management By default, ccache has a 5 GB limit on the total size of files in the cache and no limit on the number of files. You can set different limits using the command -line options *-M*/*--max-size* and *-F*/*--max-files*. Use *ccache --s/--show-stats* to see the cache size and the currently configured limits (in -addition to other various statistics). +line options `-M`/`--max-size` and `-F`/`--max-files`. Use the +`-s`/`--show-stats` option to see the cache size and the currently configured +limits (in addition to other various statistics). Cleanup can be triggered in two different ways: automatic and manual. -Automatic cleanup -~~~~~~~~~~~~~~~~~ +=== Automatic cleanup Ccache maintains counters for various statistics about the cache, including the size and number of all cached files. In order to improve performance and reduce @@ -912,10 +1198,9 @@ idea to trigger it often, like after each cache miss. -Manual cleanup -~~~~~~~~~~~~~~ +=== Manual cleanup -You can run *ccache -c/--cleanup* to force cleanup of the whole cache, i.e. all +You can run `ccache -c/--cleanup` to force cleanup of the whole cache, i.e. all of the sixteen subdirectories. This will recalculate the statistics counters and make sure that the configuration options *max_size* and <> are not exceeded. Note that @@ -923,8 +1208,7 @@ cleanup. -Cache compression ------------------ +== Cache compression Ccache will by default compress all data it puts into the cache using the compression algorithm http://zstd.net[Zstandard] (zstd) using compression level @@ -935,24 +1219,24 @@ <> and <> for more information. -You can use the command line option *-x/--show-compression* to print +You can use the command line option `-x`/`--show-compression` to print information related to compression. Example: ------------------------------------------------------------------------------- -Total data: 14.8 GB (16.0 GB disk blocks) -Compressed data: 11.3 GB (30.6% of original size) - - Original data: 36.9 GB - - Compression ratio: 3.267 x (69.4% space savings) -Incompressible data: 3.5 GB +Total data: 14.8 GB (16.0 GB disk blocks) +Compressed data: 11.3 GB (30.6% of original size) + Original size: 36.9 GB + Compression ratio: 3.267 x (69.4% space savings) +Incompressible data: 3.5 GB ------------------------------------------------------------------------------- Notes: -* The ``disk blocks'' size is the cache size when taking disk block size into - account. This value should match the ``cache size'' value from ``ccache - --show-stats''. The other size numbers refer to actual content sizes. -* ``Compressed data'' refers to result and manifest files stored in the cache. -* ``Incompressible data'' refers to files that are always stored uncompressed +* The "`disk blocks`" size is the cache size when taking disk block size into + account. This value should match the "`Cache size`" value from "`ccache + --show-stats`". The other size numbers refer to actual content sizes. +* "`Compressed data`" refers to result and manifest files stored in the cache. +* "`Incompressible data`" refers to files that are always stored uncompressed (triggered by enabling <> or <>) or unknown files (for instance files created by older ccache versions). @@ -960,7 +1244,7 @@ <>. The cache data can also be recompressed to another compression level (or made -uncompressed) with the command line option *-X/--recompress*. If you choose to +uncompressed) with the command line option `-X`/`--recompress`. If you choose to disable compression by default or to use a low compression level, you can (re)compress newly cached data with a higher compression level after the build or at another time when there are more CPU cycles available, for instance every @@ -969,130 +1253,114 @@ recompressed. -Cache statistics ----------------- +== Cache statistics -*ccache -s/--show-stats* can show the following statistics: +`ccache --show-stats` shows a summary of statistics, including cache size, +cleanups (number of performed cleanups, either implicitly due to a cache size +limit being reached or due to explicit `ccache -c` calls), overall hit rate, hit +rate for <>/<> modes +and hit rate for primary and <> storage. + +The summary also includes counters called "`Errors`" and "`Uncacheable`", which +are sums of more detailed counters. To see those detailed counters, use the +`-v`/`--verbose` flag. The verbose mode can show the following counters: [options="header",cols="30%,70%"] |============================================================================== -|Name | Description -| autoconf compile/link | -Uncachable compilation or linking by an autoconf test. +| *Counter* | *Description* -| bad compiler arguments | +| Autoconf compile/link | +Uncacheable compilation or linking by an Autoconf test. + +| Bad compiler arguments | Malformed compiler argument, e.g. missing a value for a compiler option that requires an argument or failure to read a file specified by a compiler option argument. -| cache file missing | -A file was unexpectedly missing from the cache. This only happens in rare -situations, e.g. if one ccache instance is about to get a file from the cache -while another instance removed the file as part of cache cleanup. - -| cache hit (direct) | -A result was successfully found using <<_the_direct_mode,the direct mode>>. - -| cache hit (preprocessed) | -A result was successfully found using <<_the_preprocessor_mode,the preprocessor -mode>>. - -| cache miss | -No result was found. - -| cache size | -Current size of the cache. - -| called for link | +| Called for linking | The compiler was called for linking, not compiling. Ccache only supports -compilation of a single file, i.e. calling the compiler with the *-c* option to +compilation of a single file, i.e. calling the compiler with the `-c` option to produce a single object file from a single source file. -| called for preprocessing | +| Called for preprocessing | The compiler was called for preprocessing, not compiling. -| can't use precompiled header | -Preconditions for using <<_precompiled_headers,precompiled headers>> were not -fulfilled. - -| can't use modules | -Preconditions for using <<_c_modules,C++ modules>> were not fulfilled. +| Could not use modules | +Preconditions for using <> were not fulfilled. -| ccache internal error | -Unexpected failure, e.g. due to problems reading/writing the cache. +| Could not use precompiled header | +Preconditions for using <> were not +fulfilled. -| cleanups performed | -Number of cleanups performed, either implicitly due to the cache size limit -being reached or due to explicit *ccache -c/--cleanup* calls. +| Could not write to output file | +The output path specified with `-o` is not a file (e.g. a directory or a device +node). -| compile failed | +| Compilation failed | The compilation failed. No result stored in the cache. -| compiler check failed | +| Compiler check failed | A compiler check program specified by <> (*CCACHE_COMPILERCHECK*) failed. -| compiler produced empty output | +| Compiler produced empty output | The compiler's output file (typically an object file) was empty after compilation. -| compiler produced no output | +| Compiler produced no output | The compiler's output file (typically an object file) was missing after compilation. -| compiler produced stdout | +| Compiler produced stdout | The compiler wrote data to standard output. This is something that compilers normally never do, so ccache is not designed to store such output in the cache. -| couldn't find the compiler | +| Could not find the compiler | The compiler to execute could not be found. -| error hashing extra file | +| Error hashing extra file | Failure reading a file specified by <> (*CCACHE_EXTRAFILES*). -| files in cache | -Current number of files in the cache. +| Forced recache | +<> was used to overwrite an existing result. + +| Internal error | +Unexpected failure, e.g. due to problems reading/writing the cache. + +| Missing cache file | +A file was unexpectedly missing from the cache. This only happens in rare +situations, e.g. if one ccache instance is about to get a file from the cache +while another instance removed the file as part of cache cleanup. -| multiple source files | +| Multiple source files | The compiler was called to compile multiple source files in one go. This is not supported by ccache. -| no input file | +| No input file | No input file was specified to the compiler. -| output to a non-regular file | -The output path specified with *-o* is not a file (e.g. a directory or a device -node). - -| output to stdout | -The compiler was instructed to write its output to standard output using *-o --*. This is not supported by ccache. +| Output to stdout | +The compiler was instructed to write its output to standard output using `-o -`. +This is not supported by ccache. -| preprocessor error | -Preprocessing the source code using the compiler's *-E* option failed. +| Preprocessing failed | +Preprocessing the source code using the compiler's `-E` option failed. -| stats updated | -When statistics were updated the last time. - -| stats zeroed | -When *ccache -z* was called the last time. - -| unsupported code directive | -Code like the assembler *.incbin* directive was found. This is not supported +| Unsupported code directive | +Code like the assembler `.incbin` directive was found. This is not supported by ccache. -| unsupported compiler option | +| Unsupported compiler option | A compiler option not supported by ccache was found. -| unsupported source language | -A source language e.g. specified with *-x* was unsupported by ccache. +| Unsupported source language | +A source language e.g. specified with `-x` was unsupported by ccache. |============================================================================== -How ccache works ----------------- +== How ccache works The basic idea is to detect when you are compiling exactly the same code a second time and reuse the previously produced output. The detection is done by @@ -1115,20 +1383,18 @@ The direct mode is generally faster since running the preprocessor has some overhead. -If no previous result is detected (i.e., there is a cache miss) using the -direct mode, ccache will fall back to the preprocessor mode unless the *depend -mode* is enabled. In the depend mode, ccache never runs the preprocessor, not -even on cache misses. Read more in _<<_the_depend_mode,The depend mode>>_ -below. +If no previous result is detected (i.e., there is a cache miss) using the direct +mode, ccache will fall back to the preprocessor mode unless the *depend mode* is +enabled. In the depend mode, ccache never runs the preprocessor, not even on +cache misses. Read more in _<>_ below. -Common hashed information -~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Common hashed information The following information is always included in the hash: * the extension used by the compiler for a file with preprocessor output - (normally *.i* for C code and *.ii* for C++ code) + (normally `.i` for C code and `.ii` for C++ code) * the compiler's size and modification time (or other compiler-specific information specified by <>) * the name of the compiler @@ -1137,14 +1403,13 @@ <> (if any) -The preprocessor mode -~~~~~~~~~~~~~~~~~~~~~ +=== The preprocessor mode In the preprocessor mode, the hash is formed of the common information and: -* the preprocessor output from running the compiler with *-E* -* the command line options except those that affect include files (*-I*, - *-include*, *-D*, etc; the theory is that these command line options will +* the preprocessor output from running the compiler with `-E` +* the command line options except those that affect include files (`-I`, + `-include`, `-D`, etc; the theory is that these command line options will change the preprocessor output if they have any effect at all) * any standard error output generated by the preprocessor @@ -1152,15 +1417,14 @@ the cache. -The direct mode -~~~~~~~~~~~~~~~ +=== The direct mode In the direct mode, the hash is formed of the common information and: * the input source file * the compiler options -Based on the hash, a data structure called ``manifest'' is looked up in the +Based on the hash, a data structure called "`manifest`" is looked up in the cache. The manifest contains: * references to cached compilation results (object file, dependency file, etc) @@ -1191,19 +1455,18 @@ * a modification time of one of the include files is too new (needed to avoid a race condition) * a compiler option not supported by the direct mode is used: -** a *-Wp,_X_* compiler option other than *-Wp,-MD,_path_*, - *-Wp,-MMD,_path_* and *-Wp,-D_define_* -** *-Xpreprocessor* -* the string `__TIME__` is present in the source code +** a `-Wp,++*++` compiler option other than `-Wp,-MD,`, `-Wp,-MMD,` + and `-Wp,-D` +** `-Xpreprocessor` +* the string `+__TIME__+` is present in the source code -The depend mode -~~~~~~~~~~~~~~~ +=== The depend mode If the depend mode is enabled, ccache will not use the preprocessor at all. The hash used to identify results in the cache will be based on the direct mode hash described above plus information about include files read from the -dependency file generated by the compiler with *-MD* or *-MMD*. +dependency file generated by the compiler with `-MD` or `-MMD`. Advantages: @@ -1228,15 +1491,14 @@ * <> is false. * <> is false. -* The compiler is not generating dependencies using *-MD* or *-MMD*. +* The compiler is not generating dependencies using `-MD` or `-MMD`. -Handling of newly created header files --------------------------------------- +== Handling of newly created header files If modification time (mtime) or status change time (ctime) of one of the include files is the same second as the time compilation is being done, ccache disables -the direct mode (or, in the case of a <<_precompiled_headers,precompiled +the direct mode (or, in the case of a <>, disables caching completely). This done as a safety measure to avoid a race condition (see below). @@ -1259,12 +1521,11 @@ 5. The wrong object file is stored in the cache. -Cache debugging ---------------- +== Cache debugging To find out what information ccache actually is hashing, you can enable the debug mode via the configuration option <> or by setting -*CCACHE_DEBUG* in the environment. This can be useful if you are investigating +`CCACHE_DEBUG` in the environment. This can be useful if you are investigating why you don't get cache hits. Note that performance will be reduced slightly. When the debug mode is enabled, ccache will create up to five additional files @@ -1272,61 +1533,62 @@ [options="header",cols="30%,70%"] |============================================================================== -|Filename | Description -| *.ccache-input-c* | +| *Filename* | *Description* + +| `.ccache-input-c` | Binary input hashed by both the direct mode and the preprocessor mode. -| *.ccache-input-d* | +| `.ccache-input-d` | Binary input only hashed by the direct mode. -| *.ccache-input-p* | +| `.ccache-input-p` | Binary input only hashed by the preprocessor mode. -| *.ccache-input-text* | +| `.ccache-input-text` | Human-readable combined diffable text version of the three files above. -| *.ccache-log* | +| `.ccache-log` | Log for this object file. |============================================================================== -If <> (environment variable *CCACHE_DEBUGDIR*) is +If <> (environment variable `CCACHE_DEBUGDIR`) is set, the files above will be written to that directory with full absolute paths instead of next to the object file. In the direct mode, ccache uses the 160 bit BLAKE3 hash of the -*ccache-input-c* + *ccache-input-d* data (where *+* means concatenation), while -the *ccache-input-c* + *ccache-input-p* data is used in the preprocessor mode. - -The *ccache-input-text* file is a combined text version of the three -binary input files. It has three sections (``COMMON'', ``DIRECT MODE'' and -``PREPROCESSOR MODE''), which is turn contain annotations that say what kind of +"`ccache-input-c`" + "`ccache-input-d`" data (where *+* means concatenation), +while the "`ccache-input-c`" + "`ccache-input-p`" data is used in the +preprocessor mode. + +The "`ccache-input-text`" file is a combined text version of the three binary +input files. It has three sections ("`COMMON`", "`DIRECT MODE`" and +"`PREPROCESSOR MODE`"), which is turn contain annotations that say what kind of data comes next. To debug why you don't get an expected cache hit for an object file, you can do something like this: 1. Build with debug mode enabled. -2. Save the *.ccache-** files. +2. Save the `.ccache-++*++` files. 3. Build again with debug mode enabled. -4. Compare *.ccache-input-text* for the two builds. This together - with the *.ccache-log* files should give you some clues about +4. Compare `.ccache-input-text` for the two builds. This together + with the `.ccache-log` files should give you some clues about what is happening. -Compiling in different directories ----------------------------------- +== Compiling in different directories Some information included in the hash that identifies a unique compilation can contain absolute paths: * The preprocessed source code may contain absolute paths to include files if - the compiler option *-g* is used or if absolute paths are given to *-I* and + the compiler option `-g` is used or if absolute paths are given to `-I` and similar compiler options. -* Paths specified by compiler options (such as *-I*, *-MF*, etc) on the command +* Paths specified by compiler options (such as `-I`, `-MF`, etc) on the command line may be absolute. * The source code file path may be absolute, and that path may substituted for - `__FILE__` macros in the source code or included in warnings emitted to + `+__FILE__+` macros in the source code or included in warnings emitted to standard error by the preprocessor. This means that if you compile the same code in different locations, you can't @@ -1337,73 +1599,66 @@ Here's what can be done to enable cache hits between different build directories: -* If you build with *-g* (or similar) to add debug information to the object +* If you build with `-g` (or similar) to add debug information to the object file, you must either: -+ --- -** use the compiler option *-fdebug-prefix-map=_old_=_new_* for relocating - debug info to a common prefix (e.g. *-fdebug-prefix-map=$PWD=.*); or +** use the compiler option `-fdebug-prefix-map==` for relocating + debug info to a common prefix (e.g. `-fdebug-prefix-map=$PWD=.`); or ** set *hash_dir = false*. --- * If you use absolute paths anywhere on the command line (e.g. the source code - file path or an argument to compiler options like *-I* and *-MF*), you must - set <> to an absolute path to a ``base - directory''. Ccache will then rewrite absolute paths under that directory to + file path or an argument to compiler options like `-I` and `-MF`), you must + set <> to an absolute path to a "`base + directory`". Ccache will then rewrite absolute paths under that directory to relative before computing the hash. -Precompiled headers -------------------- +== Precompiled headers Ccache has support for GCC's precompiled headers. However, you have to do some things to make it work properly: * You must set <> to *pch_defines,time_macros*. - The reason is that ccache can't tell whether `__TIME__`, `__DATE__` or - `__TIMESTAMP__` is used when using a precompiled header. Further, it can't - detect changes in **#define**s in the source code because of how - preprocessing works in combination with precompiled headers. + The reason is that ccache can't tell whether `+__TIME__+`, `+__DATE__+` or + `+__TIMESTAMP__+` is used when using a precompiled header. Further, it can't + detect changes in ``#define``s in the source code because of how preprocessing + works in combination with precompiled headers. * You may also want to include *include_file_mtime,include_file_ctime* in <>. See - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. + _<>_. * You must either: + -- -** use the compiler option *-include* to include the precompiled header (i.e., - don't use *#include* in the source code to include the header; the filename - itself must be sufficient to find the header, i.e. *-I* paths are not +* use the compiler option `-include` to include the precompiled header (i.e., + don't use `#include` in the source code to include the header; the filename + itself must be sufficient to find the header, i.e. `-I` paths are not searched); or -** (for the Clang compiler) use the compiler option *-include-pch* to include +* (for the Clang compiler) use the compiler option `-include-pch` to include the PCH file generated from the precompiled header; or -** (for the GCC compiler) add the compiler option *-fpch-preprocess* when +* (for the GCC compiler) add the compiler option `-fpch-preprocess` when compiling. - -If you don't do this, either the non-precompiled version of the header file -will be used (if available) or ccache will fall back to running the real -compiler and increase the statistics counter ``preprocessor error'' (if the -non-precompiled header file is not available). -- ++ +If you don't do this, either the non-precompiled version of the header file will +be used (if available) or ccache will fall back to running the real compiler and +increase the statistics counter "`Preprocessing failed`" (if the non-precompiled +header file is not available). -C++ modules ------------ +== C++ modules -Ccache has support for Clang's *-fmodules* option. In practice ccache only -additionally hashes *module.modulemap* files; it does not know how Clang +Ccache has support for Clang's `-fmodules` option. In practice ccache only +additionally hashes `module.modulemap` files; it does not know how Clang handles its cached binary form of modules so those are ignored. This should not -matter in practice: as long as everything else (including *module.modulemap* +matter in practice: as long as everything else (including `module.modulemap` files) is the same the cached result should work. Still, you must set <> to *modules* to allow caching. -You must use both <<_the_direct_mode,*direct mode*>> and -<<_the_depend_mode,*depend mode*>>. When using <<_the_preprocessor_mode,the -preprocessor mode>> Clang does not provide enough information to allow hashing -of *module.modulemap* files. +You must use both <> and +<>. When using +<> Clang does not provide enough +information to allow hashing of `module.modulemap` files. -Sharing a cache ---------------- +== Sharing a cache A group of developers can increase the cache hit rate by sharing a cache directory. To share a cache without unpleasant side effects, the following @@ -1413,7 +1668,7 @@ * Make sure that the configuration option <> is false (which is the default). * Make sure that all users are in the same group. -* Set the configuration option <> to 002. This ensures +* Set the configuration option <> to *002*. This ensures that cached files are accessible to everyone in the group. * Make sure that all users have write permission in the entire cache directory (and that you trust all users of the shared cache). @@ -1421,11 +1676,9 @@ tells the filesystem to inherit group ownership for new directories. The following command might be useful for this: + --- ---- find $CCACHE_DIR -type d | xargs chmod g+s ---- --- The reason to avoid the hard link mode is that the hard links cause unwanted side effects, as all links to a cached file share the file's modification @@ -1438,8 +1691,7 @@ discussed in a previous section. -Sharing a cache on NFS ----------------------- +== Sharing a cache on NFS It is possible to put the cache directory on an NFS filesystem (or similar filesystems), but keep in mind that: @@ -1458,26 +1710,27 @@ <> to *system_headers* to ignore system headers. +An alternative to putting the main cache directory on NFS is to set up a +<> file cache. + -Using ccache with other compiler wrappers ------------------------------------------ +== Using ccache with other compiler wrappers The recommended way of combining ccache with another compiler wrapper (such as -``distcc'') is by letting ccache execute the compiler wrapper. This is +"`distcc`") is by letting ccache execute the compiler wrapper. This is accomplished by defining <>, for -example by setting the environment variable *CCACHE_PREFIX* to the name of the -wrapper (e.g. *distcc*). Ccache will then prefix the command line with the +example by setting the environment variable `CCACHE_PREFIX` to the name of the +wrapper (e.g. `distcc`). Ccache will then prefix the command line with the specified command when running the compiler. To specify several prefix commands, set <> to a colon-separated list of commands. Unless you set <> to a suitable command -(see the description of that configuration option), it is not recommended to -use the form *ccache anotherwrapper compiler args* as the compilation command. -It's also not recommended to use the masquerading technique for the other -compiler wrapper. The reason is that by default, ccache will in both cases hash -the mtime and size of the other wrapper instead of the real compiler, which -means that: +(see the description of that configuration option), it is not recommended to use +the form `ccache anotherwrapper compiler args` as the compilation command. It's +also not recommended to use the masquerading technique for the other compiler +wrapper. The reason is that by default, ccache will in both cases hash the mtime +and size of the other wrapper instead of the real compiler, which means that: * Compiler upgrades will not be detected properly. * The cached results will not be shared between compilations with and without @@ -1487,31 +1740,27 @@ used, ccache will not invoke the other wrapper when running the preprocessor, which increases performance. You can use <> if you also want to invoke -the other wrapper when doing preprocessing (normally by adding *-E*). +the other wrapper when doing preprocessing (normally by adding `-E`). -Caveats -------- +== Caveats * The direct mode fails to pick up new header files in some rare scenarios. See - _<<_the_direct_mode,The direct mode>>_ above. + _<>_ above. -Troubleshooting ---------------- +== Troubleshooting -General -~~~~~~~ +=== General A general tip for getting information about what ccache is doing is to enable debug logging by setting the configuration option <> (or -the environment variable *CCACHE_DEBUG*); see _<<_cache_debugging,Cache -debugging>>_ for more information. Another way of keeping track of what is +the environment variable *CCACHE_DEBUG*); see _<>_ +for more information. Another way of keeping track of what is happening is to check the output of *ccache -s*. -Performance -~~~~~~~~~~~ +=== Performance Ccache has been written to perform well out of the box, but sometimes you may have to do some adjustments of how you use the compiler and ccache in order to @@ -1521,77 +1770,74 @@ storage device if possible. Having lots of free memory so that files in the cache directory stay in the disk cache is also preferable. -A good way of monitoring how well ccache works is to run *ccache -s* before and +A good way of monitoring how well ccache works is to run `ccache -s` before and after your build and then compare the statistics counters. Here are some common problems and what may be done to increase the hit rate: -* If ``cache hit (preprocessed)'' has been incremented instead of ``cache hit - (direct)'', ccache has fallen back to preprocessor mode, which is generally - slower. Some possible reasons are: +* If the counter for preprocessed cache hits has been incremented instead of the + one for direct cache hits, ccache has fallen back to preprocessor mode, which + is generally slower. Some possible reasons are: ** The source code has been modified in such a way that the preprocessor output is not affected. ** Compiler arguments that are hashed in the direct mode but not in the - preprocessor mode have changed (*-I*, *-include*, *-D*, etc) and they didn't + preprocessor mode have changed (`-I`, `-include`, `-D`, etc) and they didn't affect the preprocessor output. -** The compiler option *-Xpreprocessor* or *-Wp,_X_* (except *-Wp,-MD,_path_*, - *-Wp,-MMD,_path_*, and *-Wp,-D_define_*) is used. +** The compiler option `-Xpreprocessor` or `-Wp,++*++` (except `-Wp,-MD,`, + `-Wp,-MMD,`, and `-Wp,-D`) is used. ** This was the first compilation with a new value of the <>. ** A modification or status change time of one of the include files is too new (created the same second as the compilation is being done). See - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. -** The `__TIME__` preprocessor macro is (potentially) being used. Ccache turns - off direct mode if `__TIME__` is present in the source code. This is done as - a safety measure since the string indicates that a `__TIME__` macro _may_ - affect the output. (To be sure, ccache would have to run the preprocessor, - but the sole point of the direct mode is to avoid that.) If you know that - `__TIME__` isn't used in practise, or don't care if ccache produces objects - where `__TIME__` is expanded to something in the past, you can set - <> to *time_macros*. -** The `__DATE__` preprocessor macro is (potentially) being used and the date - has changed. This is similar to how `__TIME__` is handled. If `__DATE__` is - present in the source code, ccache hashes the current date in order to be - able to produce the correct object file if the `__DATE__` macro affects the - output. If you know that `__DATE__` isn't used in practise, or don't care if - ccache produces objects where `__DATE__` is expanded to something in the + _<>_. +** The `+__TIME__+` preprocessor macro is (potentially) being used. Ccache turns + off direct mode if `+__TIME__+` is present in the source code. This is done + as a safety measure since the string indicates that a `+__TIME__+` macro + _may_ affect the output. (To be sure, ccache would have to run the + preprocessor, but the sole point of the direct mode is to avoid that.) If you + know that `+__TIME__+` isn't used in practise, or don't care if ccache + produces objects where `+__TIME__+` is expanded to something in the past, you + can set <> to *time_macros*. +** The `+__DATE__+` preprocessor macro is (potentially) being used and the date + has changed. This is similar to how `+__TIME__+` is handled. If `+__DATE__+` + is present in the source code, ccache hashes the current date in order to be + able to produce the correct object file if the `+__DATE__+` macro affects the + output. If you know that `+__DATE__+` isn't used in practise, or don't care + if ccache produces objects where `+__DATE__+` is expanded to something in the past, you can set <> to *time_macros*. -** The `__TIMESTAMP__` preprocessor macro is (potentially) being used and the +** The `+__TIMESTAMP__+` preprocessor macro is (potentially) being used and the source file's modification time has changed. This is similar to how - `__TIME__` is handled. If `__TIMESTAMP__` is present in the source code, + `+__TIME__+` is handled. If `+__TIMESTAMP__+` is present in the source code, ccache hashes the string representation of the source file's modification time in order to be able to produce the correct object file if the - `__TIMESTAMP__` macro affects the output. If you know that `__TIMESTAMP__` - isn't used in practise, or don't care if ccache produces objects where - `__TIMESTAMP__` is expanded to something in the past, you can set - <> to *time_macros*. + `+__TIMESTAMP__+` macro affects the output. If you know that + `+__TIMESTAMP__+` isn't used in practise, or don't care if ccache produces + objects where `+__TIMESTAMP__+` is expanded to something in the past, you can + set <> to *time_macros*. ** The input file path has changed. Ccache includes the input file path in the direct mode hash to be able to take relative include files into account and - to produce a correct object file if the source code includes a `__FILE__` + to produce a correct object file if the source code includes a `+__FILE__+` macro. -* If ``cache miss'' has been incremented even though the same code has been +* If a cache hit counter was not incremented even though the same code has been compiled and cached before, ccache has either detected that something has changed anyway or a cleanup has been performed (either explicitly or - implicitly when a cache limit has been reached). Some perhaps unobvious - things that may result in a cache miss are usage of `__TIME__`, `__DATE__` or - `__TIMESTAMP__` macros, or use of automatically generated code that contains + implicitly when a cache limit has been reached). Some perhaps unobvious things + that may result in a cache miss are usage of `+__TIME__+`, `+__DATE__+` or + `+__TIMESTAMP__+` macros, or use of automatically generated code that contains a timestamp, build counter or other volatile information. -* If ``multiple source files'' has been incremented, it's an indication that - the compiler has been invoked on several source code files at once. Ccache - doesn't support that. Compile the source code files separately if possible. -* If ``unsupported compiler option'' has been incremented, enable debug logging +* If "`Multiple source files`" has been incremented, it's an indication that the + compiler has been invoked on several source code files at once. Ccache doesn't + support that. Compile the source code files separately if possible. +* If "`Unsupported compiler option`" has been incremented, enable debug logging and check which compiler option was rejected. -* If ``preprocessor error'' has been incremented, one possible reason is that - precompiled headers are being used. See _<<_precompiled_headers,Precompiled - headers>>_ for how to remedy this. -* If ``can't use precompiled header'' has been incremented, see - _<<_precompiled_headers,Precompiled headers>>_. -* If ``can't use modules'' has been incremented, see _<<_c_modules,C++ - modules>>_. +* If "`Preprocessing failed`" has been incremented, one possible reason is that + precompiled headers are being used. See _<>_ for how to + remedy this. +* If "`Could not use precompiled header`" has been incremented, see + _<>_. +* If "`Could not use modules`" has been incremented, see _<>_. -Corrupt object files -~~~~~~~~~~~~~~~~~~~~ +=== Corrupt object files It should be noted that ccache is susceptible to general storage problems. If a bad object file sneaks into the cache for some reason, it will of course stay @@ -1602,9 +1848,9 @@ 1. Build so that the bad object file ends up in the build tree. 2. Remove the bad object file from the build tree. -3. Rebuild with *CCACHE_RECACHE* set. +3. Rebuild with `CCACHE_RECACHE` set. -An alternative is to clear the whole cache with *ccache -C* if you don't mind +An alternative is to clear the whole cache with `ccache -C` if you don't mind losing other cached results. There are no reported issues about ccache producing broken object files @@ -1612,15 +1858,13 @@ case, please report it. -More information ----------------- +== More information Credits, mailing list information, bug reporting instructions, source code, etc, can be found on ccache's web site: . -Author ------- +== Author Ccache was originally written by Andrew Tridgell and is currently developed and maintained by Joel Rosdahl. See AUTHORS.txt or AUTHORS.html and diff -Nru ccache-4.2.1/doc/NEWS.adoc ccache-4.5.1/doc/NEWS.adoc --- ccache-4.2.1/doc/NEWS.adoc 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/doc/NEWS.adoc 2021-11-17 19:31:58.000000000 +0000 @@ -1,16 +1,378 @@ -Ccache news -=========== += Ccache news + +== Ccache 4.5.1 + +Release date: 2021-11-17 + + +=== Bug fixes + +- Fixed entry_size field for result entries. This bug affected the recompression + feature (`-X`/`--recompress`) in ccache 4.5. + + [small]#_[contributed by Joel Rosdahl]_# + +- The actual compression level is now once again stored in the cache entry + header. + + [small]#_[contributed by Joel Rosdahl]_# + +- Corrected error handling for unconstructible secondary storage backends. For + instance, this avoids a crash when a Redis server can't be reached. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.5 + +Release date: 2021-11-13 + + +=== New features + +- Made various improvements to the cache entry format. Among other things, the + header now contains a creation timestamp and a note of the ccache version used + to create the entry. The cache entry checksum has also been upgraded to use + 128-bit XXH3 instead 64-bit XXH3. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added support for cache namespaces. If a namespace configured, e.g. using + `CCACHE_NAMESPACE=some_string`, the namespace string will be added to the + hashed data for each compilation. This will make the associated cache entries + logically separate from cache entries in other namespaces, but they will still + share the same storage space. Cache entries can also be selectively removed + from the primary cache with the new command line option `--evict-namespace`, + potentially in combination with `--evict-older-than`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made HTTP keep-alive configurable, defaulting to off for now. + + [small]#_[contributed by Gregor Jasny]_# + +- Added support for rewriting absolute path to Clang option `--gcc-toolchain`. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Compatibility notes + +- A consequence of the changed cache entry format is that ccache 4.5 will not + share cache entries with earlier versions. Different ccache versions can + however still use the same cache storage without any issues. + + +=== Bug fixes + +- Fixed a problem with special characters in the user info part of URLs for HTTP + storage. + + [small]#_[contributed by Russell McClellan]_# + +- Fixed win32 log messages about file locks. + + [small]#_[contributed by Luboš Luňák]_# + +- Fixed debug directory handling on Windows. + + [small]#_[contributed by Luboš Luňák]_# + +- The hard link and file clone features are now disabled when secondary storage + is used since they only work for the local primary cache. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.4.2 + +Release date: 2021-09-28 + + +=== Bug fixes + +- Fixed a bug introduced in 4.4 where ccache could produce false direct cache + hits in some situations if it decides to disable the direct mode temporarily + (e.g. due to "`too new header`" file). + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Test improvements + +- Use shell builtin pwd command for basedir test. + + [small]#_[contributed by Kira Bruneau]_# + +- Cope with CC being a wrapper script that uses ccache. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.4.1 + +Release date: 2021-09-11 + + +=== New features + +- The secondary storage statistics section of `-s/--show-stats` is now shown + only if it's non-empty or with two verbose options. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added display of statistics counters for misses. Previously they were only + implicit in the "`hits + misses`" sums. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- Fixed spurious crashes when using the HTTP or Redis backend and the remote + connection hung up. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made sure to always store configuration origin value. + + [small]#_[contributed by Gregor Jasny]_# + + +=== Build improvements + +- The matching version of lld is now used for Clang. + + [small]#_[contributed by Gregor Jasny]_# + +- The standard linker is now used if IPO (LTO) is enabled. + + [small]#_[contributed by Gregor Jasny]_# + +- Disabled IPO (LTO) for MinGW toolchains since they seem to be generally + broken. + + [small]#_[contributed by Gregor Jasny]_# + +- Fixed build errors with Clang on Windows. + + [small]#_[contributed by Orgad Shaneh]_# + + +=== Test improvements + +- Fixed .incbin test with newer binutil versions. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed basedir test suite failure when using a symlinked CWD. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved output of differing text files on failure. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.4 + +Release date: 2021-08-19 + + +=== New features + +- Made it possible to share a cache over network or on a local filesystem. The + configuration option `secondary_storage`/`CCACHE_SECONDARY_STORAGE` specifies + one or several storage backends to query after the primary local cache + storage. It is also possible to configure sharding (partitioning) of the cache + to spread it over a server cluster using + https://en.wikipedia.org/wiki/Rendezvous_hashing[Rendezvous hashing]. See the + _https://ccache.dev/manual/4.4.html#_secondary_storage_backends[Secondary + storage backends]_ chapter in the manual for details. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added an HTTP backend for secondary storage on any HTTP server that supports + GET/PUT/DELETE methods. See https://ccache.dev/howto/http-storage.html[How to + set up HTTP storage] for hints on how to set up an HTTP server for use with + ccache. + + [small]#_[contributed by Gregor Jasny]_# + +- Added a Redis backend for secondary storage on any server that supports the + Redis protocol. See https://ccache.dev/howto/redis-storage.html[How to set up + Redis storage] for hints on how to set up a Redis server for use with + ccache. + + [small]#_[contributed by Anders F Björklund]_# + +- Added a filesystem backend for secondary storage. It can for instance be used + for a shared cache over networked filesystems such as NFS, or for mounting a + secondary read-only cache layer into a build container. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added `--trim-dir`, `--trim-max-size` and `--trim-method` options that can be + used to trim a secondary storage directory to a certain size, e.g. via + cron. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a configuration option `reshare`/`CCACHE_RESHARE` which makes ccache + send results to secondary storage even for primary storage cache hits. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added new statistics counters for direct/preprocessed cache misses, primary + storage hits/misses, secondary storage hits/misses/errors/timeouts and forced + recaches. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved statistics summary. The `-s`/`--show-stats` option now prints a more + condensed overview where the counters representing "`uncacheable calls`" are + summed as uncacheable and errors counters. The summary shows hit rate for + direct/preprocessed hits/misses, as well as primary/secondary storage + hits/misses. More details are shown with `-v`/`--verbose`. Note: Scripts + should use `--print-stats` (available since ccache 3.7) instead of trying to + parse the output of `--show-stats`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a "`stats log`" feature (configuration option + `stats_log`/`CCACHE_STATSLOG`), which tells ccache to store statistics in a + separate log file specified by the user. It can for instance be used to + collect statistics for a single build without interference from other + concurrent builds. Statistics from the log file can then be viewed with + `ccache --show-log-stats`. + + [small]#_[contributed by Anders F Björklund]_# + +- Added support for clang's `--config` option. + + [small]#_[contributed by Tom Stellard]_# + +- Added support for one `-Xarch_*` option that matches a corresponding `-arch` + option. + + [small]#_[contributed by Joel Rosdahl]_# + +- Renamed the `--directory` option to `--dir` for consistency with other + options. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made the `--config-path` and `--dir` options affect the whole command line so + that they don't have to be put before `-s`/`--show-stats`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made `--dump-manifest` and `--dump-result` accept filename `-` for reading + from standard input. + + [small]#_[contributed by Anders F Björklund]_# + +- Made the output of `--print-stats` sorted. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added more internal trace points. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- Fixed a crash if using `base_dir` and `$PWD` is set to a relative path. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed a bug with `-fprofile-generate` where ccache could give false positive + cache hits when compiling with relative paths in another directory. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed a bug in `debug_dir`/`CCACHE_DEBUGDIR`. The absolute path to the object + file was not created correctly if the object file didn't already exist. + + [small]#_[contributed by Joel Rosdahl]_# + +- Disabled preprocessor hits for pre-compiled headers with Clang again. + + [small]#_[contributed by Arne Hasselbring]_# + +- Fixed a problem when using the Gold linker on MIPS by only probing for a + faster linker in dev build mode and on x86_64. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made the `-DENABLE_TRACING=1` mode work again. + + [small]#_[contributed by Anders F Björklund]_# + + +=== Changed tooling + +- A C++14 compiler or newer is now required to build ccache. For GCC, this means + version 6 or newer in practice. + +- CMake 3.10 or newer is now required to build ccache. + +- https://asciidoctor.org[Asciidoctor] is now required to build ccache + documentation. + + +=== Build/test/documentation improvements + +- Fixed an issue in the modules test suite that showed up when running the + ccache test suite with the clang wrapper provided by Nixpkgs. + + [small]#_[contributed by Ryan Burns]_# + +- Made the nvcc_ldir test suite require a working NVCC. + + [small]#_[contributed by Michael Kruse]_# + +- Made the ivfsoverlay test suite more robust. + + [small]#_[contributed by Michael Kruse]_# + +- Fixed issue with usage of `/FI` when building ccache with MSVC. + + [small]#_[contributed by Michael Kruse]_# + +- Fixed Apple Clang detection in the integration test suite. + + [small]#_[contributed by Gregor Jasny]_# + +- Made clang the default compiler when running the test suite on macOS. + + [small]#_[contributed by Gregor Jasny]_# + +- Silenced stray printout from "-P -c" test case. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed selection of the ccache binary to use when running the test suite with + multi-config generators like Xcode. + + [small]#_[contributed by Gregor Jasny]_# + +- Fixed CMake feature detection for `ctim`/`mtim` fields in `struct stat`. + + [small]#_[contributed by Gregor Jasny]_# + +- Fixed issue with not linking to .lib correctly on Windows. + + [small]#_[contributed by R. Voggenauer]_# + +- Made it possible to override `CCACHE_DEV_MODE` on the command line. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved HTML documentation style. + + [small]#_[contributed by Joel Rosdahl with minor fixes by Orgad Shaneh]_# + + +== Ccache 4.3 + +Release date: 2021-05-09 + + +=== New features + +- Ccache now ignores the Clang compiler option `-ivfsoverlay` and its argument + if you opt in to "`ivfsoverlay sloppiness`". This is useful if you use Xcode, + which uses a virtual file system (VFS) for things like combining Objective-C + and Swift code. + +- When using `-P` in combination with `-E`, ccache now reports this as "`called + for preprocessing`" instead of "`unsupported compiler option`". + +- Added support for `-specs file.specs` and `--specs file.specs` without an + equal sign between the arguments. + + +=== Bug fixes + +- "`Split dwarf`" code paths are now disabled when outputting to `/dev/null`. This + avoids an attempt to delete `/dev/null.dwo`. + +- Made the `stat`/`lstat` wrapper function for Windows treat pending deletes as + missing files. + +- Fixed a bug that made ccache process header files redundantly for some + relative headers when using Clang. + +- The object path is now included in the input hash when using `-fprofile-arcs` + (or `--coverage`) since the object file embeds a `.gcda` path based on the + object file path. + + +=== Build improvements + +- Added an `ENABLE_DOCUMENTATION` build option (default: true) that can be used + to disable the build of documentation. + +- Fixed detection of pthread features. + +- Quote CMake variables expansions to avoid errors when + `${CMAKE_C_FLAGS_RELWITHDEBINFO}` or `${CMAKE_CXX_FLAGS_RELWITHDEBINFO}` + expands to the empty string. + + +== Ccache 4.2.1 -Ccache 4.2.1 ------------- Release date: 2021-03-27 -Bug fixes -~~~~~~~~~ -- Ccache now only `dup2`s stderr into `$UNCACHED_ERR_FD` for calls to the - preprocessor/compiler. This works around a complex bug in interaction with GNU - Make, LTO linking and the Linux PTY driver. +=== Bug fixes + +- Ccache now only duplicates the stderr file descriptor into `$UNCACHED_ERR_FD` + for calls to the preprocessor/compiler. This works around a complex bug in + interaction with GNU Make, LTO linking and the Linux PTY driver. - Fixed detection of color diagnostics usage when using `-Xclang -fcolor-diagnostics` options. @@ -38,8 +400,7 @@ - Fixed handling of long command lines on Windows. -Portability and build improvements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Portability and build improvements - Build configuration scripts now probe for atomic increment as well. This fixes a linking error on Sparc. @@ -49,29 +410,28 @@ - Added support for building ccache with xlclang++ on AIX 7.2. -- Fixed assertion in the "Debug option" test. +- Fixed assertion in the "`Debug option`" test. - Made build configuration skip using ccache when building with MSVC. - Upgraded to doctest 2.4.6. This fixes a build error with glibc >= 2.34. -Documentation -~~~~~~~~~~~~~ +=== Documentation -- Fixed markup of `compiler_type` value "other". +- Fixed markup of `compiler_type` value `other`. - Fixed markup of `debug_dir` documentation. - Fixed references to the `extra_files_to_hash` configuration option. -Ccache 4.2 ----------- +== Ccache 4.2 + Release date: 2021-02-02 -New features -~~~~~~~~~~~~ + +=== New features - Improved calculation of relative paths when using `base_dir` to also consider canonical paths (i.e. paths with dereferenced symlinks) as candidates. @@ -88,8 +448,7 @@ `SOURCE_DATE_EPOCH`. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a bug where a non-Clang compiler would silently accept the Clang-specific `-f(no-)color-diagnostics` option when run via ccache. This @@ -116,8 +475,7 @@ - Fixed retrieval of the object file the destination is `/dev/null`. -Portability and build improvements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Portability and build improvements - Additional compiler flags like `-Wextra -Werror` are now only added when building ccache in developer mode. @@ -158,8 +516,7 @@ - Took steps towards being able to run the test suite on Windows. -Documentation -~~~~~~~~~~~~~ +=== Documentation - Improved wording of `compiler_check` string values. @@ -176,16 +533,16 @@ - Mention that ccache requires the `-c` compiler option. -Ccache 4.1 ----------- +== Ccache 4.1 + Release date: 2020-11-22 -New features -~~~~~~~~~~~~ + +=== New features - Symlinks are now followed when guessing the compiler. This makes ccache able - to guess compiler type “GCC” for a common symlink chain like this: - `/usr/bin/cc` → `/etc/alternatives/cc` → `/usr/bin/gcc` → `gcc-9` → + to guess compiler type "`GCC`" for a common symlink chain like this: + `/usr/bin/cc` -> `/etc/alternatives/cc` -> `/usr/bin/gcc` -> `gcc-9` -> `x86_64-linux-gnu-gcc-9`. - Added a new `compiler_type` (`CCACHE_COMPILERTYPE`) configuration option that @@ -198,20 +555,19 @@ `CCACHE_CONFIGPATH` temporarily. -Bug fixes -~~~~~~~~~ +=== Bug fixes - The original color diagnostics options are now retained when forcing colored output. This fixes a bug where feature detection of the `-fcolor-diagnostics` option would succeed when run via ccache even though the actual compiler - doesn’t support it (e.g. GCC <4.9). + doesn't support it (e.g. GCC <4.9). - Fixed a bug related to umask when using the `umask` (`CCACHE_UMASK`) configuration option. - Allow `ccache ccache compiler ...` (repeated `ccache`) again. -- Fixed parsing of dependency file in the “depend mode” so that filenames with +- Fixed parsing of dependency file in the "`depend mode`" so that filenames with space or other special characters are handled correctly. - Fixed rewriting of the dependency file content when the object filename @@ -224,16 +580,15 @@ found out at runtime that file cloning is unsupported by the OS. -Portability and build fixes -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Portability and build fixes - The ccache binary is now linked with `libatomic` if needed. This fixes build problems with GCC on ARM and PowerPC. - Fixed build of BLAKE3 code with Clang 3.4 and 3.5. -- Fixed “use of undeclared identifier 'CLONE_NOOWNERCOPY'” build error on macOS - 10.12. +- Fixed "`use of undeclared identifier 'CLONE_NOOWNERCOPY'`" build error on + macOS 10.12. - Fixed build problems related to missing AVX2 and AVX512 support on older macOS versions. @@ -241,7 +596,7 @@ - Fixed static linkage with libgcc and libstdc++ for MinGW and made it optional. -- Fixed conditional compilation of “robust mutex” code for the inode cache +- Fixed conditional compilation of "`robust mutex`" code for the inode cache routines. - Fixed badly named man page filename (`Ccache.1` instead of `ccache.1`). @@ -249,8 +604,7 @@ - Disabled some tests on ancient Clang versions. -Other improvements and fixes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Other improvements and fixes - The man page is now built by default if the required tools are available. @@ -258,20 +612,19 @@ - Improved build errors when building ccache with very old compiler versions. -- Fall back to version “unknown” when Git is not installed. +- Fall back to version "`unknown`" when Git is not installed. - Documented the relationship between `CCACHE_DIR` and `-d/--directory`. - Fixed incorrect reference and bad markup in the manual. -Ccache 4.0 ----------- +== Ccache 4.0 + Release date: 2020-10-18 -Summary of major changes -~~~~~~~~~~~~~~~~~~~~~~~~ +=== Summary of major changes - Changed the default cache directory location to follow the XDG base directory specification. @@ -288,31 +641,29 @@ - Improved cache directory structure. -- Added support for using file cloning (AKA “reflinks”). +- Added support for using file cloning (AKA "`reflinks`"). -- Added an experimental “inode cache” for file hashes. +- Added an experimental "`inode cache`" for file hashes. -Compatibility notes -~~~~~~~~~~~~~~~~~~~ +=== Compatibility notes - The default location of the cache directory has changed to follow the XDG - base directory specification (<<_detailed_functional_changes,more details + base directory specification (<>). This means that scripts can no longer assume that the cache directory is `~/.ccache` by default. The `CCACHE_DIR` environment variable still overrides the default location just like before. - The cache directory structure has changed compared to previous versions - (<<_detailed_functional_changes,more details below>>). This means that ccache + (<>). This means that ccache 4.0 will not share cache results with earlier versions. It is however safe to run ccache 4.0 and earlier versions against the same cache directory: cache bookkeeping, statistics and cleanup are backward compatible, with the minor - exception that some statistics counters incremented by ccache 4.0 won’t be + exception that some statistics counters incremented by ccache 4.0 won't be visible when running `ccache -s` with an older version. -Changed tooling -~~~~~~~~~~~~~~~ +=== Changed tooling - CMake is now used instead of Autoconf for configuration and building. @@ -322,10 +673,9 @@ - Ccache can now be built using Microsoft Visual C++. -Detailed functional changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Detailed functional changes -- All data of a cached result is now stored in a single file called “result” +- All data of a cached result is now stored in a single file called "`result`" instead of up to seven files. This reduces inode usage and improves data locality. @@ -361,12 +711,12 @@ The `cache_dir_levels` (`CCACHE_NLEVELS`) configuration option has therefore been removed. -- Added an experimental “inode cache” for file hashes, allowing computed hash +- Added an experimental "`inode cache`" for file hashes, allowing computed hash values to be reused both within and between builds. The inode cache is off by default but can be enabled by setting `inode_cache` (`CCACHE_INODECACHE`) to `true`. -- Added support for using file cloning (AKA “reflinks”) on Btrfs, XFS and APFS +- Added support for using file cloning (AKA "`reflinks`") on Btrfs, XFS and APFS to copy data to and from the cache very efficiently. - Two measures have been implemented to make the hard link mode safer: @@ -416,8 +766,8 @@ - Added optional logging to syslog if `log_file` (`CCACHE_LOGFILE`) is set to `syslog`. -- The compiler option `-fmodules` is now handled in the “depend mode”. If - “depend mode” is disabled the option is still considered too hard and ccache +- The compiler option `-fmodules` is now handled in the "`depend mode`". If + "`depend mode`" is disabled the option is still considered too hard and ccache will fall back to running the compiler. - Ccache can now cache compilations with coverage notes (`.gcno` files) @@ -442,12 +792,12 @@ - Ccache is now able to share cache entries for different object file names when using `-MD` or `-MMD`. -- Clang’s `-Xclang` (used by CMake for precompiled headers), +- Clang's `-Xclang` (used by CMake for precompiled headers), `-fno-pch-timestamp`, `-emit-pch`, `-emit-pth` and `-include-pth` options are now understood. -- Added support for the HIP (“C++ Heterogeneous-Compute Interface for - Portability”) language. +- Added support for the HIP ("`C++ Heterogeneous-Compute Interface for + Portability`") language. - The manifest format now allows for header files larger than 4 GiB. @@ -464,8 +814,8 @@ - Made handling of `.dwo` files and interaction between `-gsplit-dwarf` and other `-g*` options more robust. -- The “couldn't find compiler” statistics counter is no longer incremented when - ccache exits with a fatal error. +- The "`couldn't find compiler`" statistics counter is no longer incremented + when ccache exits with a fatal error. - Failure to run a `compiler_check` command is no longer a fatal error. @@ -488,8 +838,7 @@ older than 3.1 (released 2010). -Other improvements -~~~~~~~~~~~~~~~~~~ +=== Other improvements - Improved help text and documentation of command line options. @@ -500,7 +849,7 @@ - Added HTML anchors to configuration options in the manual so that it is possible link to a specific option. -- Tweaked placement of “(readonly)” in output of `ccache -s`. +- Tweaked placement of "`(readonly)`" in output of `ccache -s`. - Improved visibility of color output from the test suite. @@ -514,13 +863,12 @@ - Disabled read-only tests on file systems that lack such support. -ccache 3.7.12 -------------- +== Ccache 3.7.12 Release date: 2020-10-01 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Coverage files (`.gcno`) produced by GCC 9+ when using `-fprofile-dir=dir` are now handled gracefully by falling back to running the compiler. @@ -529,8 +877,7 @@ 32-bit mode. -Other -~~~~~ +=== Other - Improved documentation about sharing a cache on NFS. @@ -539,35 +886,33 @@ - Fixed test case failures with GCC 4.4. -ccache 3.7.11 -------------- +== Ccache 3.7.11 Release date: 2020-07-21 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Added knowledge about `-fprofile-{correction,reorder-functions,values}`. - ccache now handles the Intel compiler option `-xCODE` (where `CODE` is a processor feature code) correctly. -- Added support for NVCC’s `-Werror` and `--Werror` options. +- Added support for NVCC's `-Werror` and `--Werror` options. -Other -~~~~~ +=== Other -- ccache’s “Directory is not hashed if using -gz[=zlib]” tests are now skipped +- ccache's "`Directory is not hashed if using -gz[=zlib]`" tests are now skipped for GCC 6. -ccache 3.7.10 ------------- +== Ccache 3.7.10 + Release date: 2020-06-22 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Improved handling of profiling options. ccache should now work correctly for profiling options like `-fprofile-{generate,use}[=path]` for GCC ≥ 9 and @@ -577,11 +922,11 @@ - ccache now copies files directly from the cache to the destination file instead of via a temporary file. This avoids problems when using filenames - long enough to be near the file system’s filename max limit. + long enough to be near the file system's filename max limit. - When the hard-link mode is enabled, ccache now only uses hard links for object files, not other files like dependency files. This is because - compilers unlink object files before writing to them but they don’t do that + compilers unlink object files before writing to them but they don't do that for dependency files, so the latter can become overwritten and therefore corrupted in the cache. @@ -592,38 +937,37 @@ - Temporary files are now deleted immediately on signals like SIGTERM and SIGINT instead of some time later in a cleanup phase. -- Fixed a bug that affected ccache’s `-o/--set-config` option for the +- Fixed a bug that affected ccache's `-o/--set-config` option for the `base_dir` and `cache_dir_levels` keys. -ccache 3.7.9 ------------- +== Ccache 3.7.9 + Release date: 2020-03-29 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed replacing of /dev/null when building as root with hard link mode enabled and using `-o /dev/null`. -- Removed incorrect assertion resulting in “ccache: error: Internal error in - format” when using `-fdebug-prefix-map=X=` with X equal to `$PWD`. +- Removed incorrect assertion resulting in "`ccache: error: Internal error in + format`" when using `-fdebug-prefix-map=X=` with X equal to `$PWD`. -Other -~~~~~ +=== Other - Improved CUDA/NVCC support: Recognize `-dc` and `-x cu` options. - Improved name of temporary file used in NFS-safe unlink. -ccache 3.7.8 ------------- +== Ccache 3.7.8 + Release date: 2020-03-16 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Use `$PWD` instead of the real CWD (current working directory) when checking for CWD in preprocessed output. This fixes a problem when `$PWD` includes a @@ -635,8 +979,7 @@ - If `localtime_r` fails the epoch time is now logged instead of garbage. -Other -~~~~~ +=== Other - Improved error message when a boolean environment variable has an invalid value. @@ -644,28 +987,28 @@ - Improved the regression fix in ccache 3.7.5 related to not passing compilation-only options to the preprocessor. -- ccache’s PCH test suite now skips running the tests if it detects broken PCH +- ccache's PCH test suite now skips running the tests if it detects broken PCH compiler support. - Fixed unit test failure on Windows. -- Fixed “stringop-truncation” build warning on Windows. +- Fixed "`stringop-truncation`" build warning on Windows. -- Improved “x_rename” implementation on Windows. +- Improved "`x_rename`" implementation on Windows. - Improved removal of temporary file when rewriting absolute paths to relative in the dependency file. -- Clarified “include_file_ctime sloppiness” in the Performance section in the +- Clarified "`include_file_ctime sloppiness`" in the Performance section in the manual. -ccache 3.7.7 ------------- +== Ccache 3.7.7 + Release date: 2020-01-05 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a bug related to object file location in the dependency file (if using `-MD` or `-MMD` but not `-MF` and the build directory is not the same as the @@ -675,32 +1018,32 @@ compilers again. (A better fix for this is planned for ccache 4.0.) - Removed the unify mode since it has bugs and shortcomings that are non-trivial - or impossible to fix: it doesn’t work with the direct mode, it doesn’t handle + or impossible to fix: it doesn't work with the direct mode, it doesn't handle C++ raw strings correctly, it can give false cache hits for `.incbin` - directives, it’s turned off when using `-g` and it can make line numbers in + directives, it's turned off when using `-g` and it can make line numbers in warning messages and `__LINE__` macros incorrect. - mtime and ctime values are now stored in the manifest files only when sloppy_file_stat is set. This avoids adding superfluous manifest file entries on direct mode cache misses. -- A “Result:” line is now always printed to the log. +- A "`Result:`" line is now always printed to the log. -- The “cache miss” statistics counter will now be updated for read-only cache +- The "`cache miss`" statistics counter will now be updated for read-only cache misses, making it consistent with the cache hit case. -ccache 3.7.6 ------------- +== Ccache 3.7.6 + Release date: 2019-11-17 -Bug fixes -~~~~~~~~~ -- The opt-in “file_macro sloppiness” mode has been removed so that the input +=== Bug fixes + +- The opt-in "`file_macro sloppiness`" mode has been removed so that the input file path now is always included in the direct mode hash. This fixes a bug - that could result in false cache hits in an edge case when “file_macro - sloppiness” is enabled and several identical source files include a relative + that could result in false cache hits in an edge case when "`file_macro + sloppiness`" is enabled and several identical source files include a relative header file with the same name but in different directories. - Statistics files are no longer lost when the filesystem of the cache is full. @@ -711,26 +1054,25 @@ - Properly handle color diagnostics in the depend mode as well. -ccache 3.7.5 ------------- +== Ccache 3.7.5 + Release date: 2019-10-22 -New features -~~~~~~~~~~~~ + +=== New features - Added support for `-MF=arg` (with an extra equal sign) as understood by EDG-based compilers. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a regression in 3.7.2 that could result in a warning message instead of - an error in an edge case related to usage of “-Werror”. + an error in an edge case related to usage of "`-Werror`". - An implicit `-MQ` is now passed to the preprocessor only if the object file extension is non-standard. This will make it easier to use EDG-based - compilers (e.g. GHS) which don’t understand `-MQ`. (This is a bug fix of the + compilers (e.g. GHS) which don't understand `-MQ`. (This is a bug fix of the corresponding improvement implemented in ccache 3.4.) - ccache now falls back to running the real compiler instead of failing fataly @@ -743,45 +1085,45 @@ - Fixed warning during configure in out-of-tree build in developer mode. -ccache 3.7.4 ------------- +== Ccache 3.7.4 + Release date: 2019-09-12 -Improvements -~~~~~~~~~~~~ + +=== Improvements - Added support for the `-gz[=type]` compiler option (previously ccache would - think that “-gz” alone would enable debug information, thus potentially + think that "`-gz`" alone would enable debug information, thus potentially including the current directory in the hash). -- Added support for converting paths like “/c/users/...” into relative paths on - Windows. +- Added support for converting paths like "`/c/users/...`" into relative paths + on Windows. + +== Ccache 3.7.3 -ccache 3.7.3 ------------- Release date: 2019-08-17 -Bug fixes -~~~~~~~~~ -- The cache size (which is counted in “used disk blocks”) is now correct on +=== Bug fixes + +- The cache size (which is counted in "`used disk blocks`") is now correct on filesystems that use more or less disk blocks than conventional filesystems, e.g. ecryptfs or btrfs/zfs with transparent compression. This also fixes a - related problem with ccache’s own test suite when run on such file systems. + related problem with ccache's own test suite when run on such file systems. + +- Fixed a regression in 3.7.2 when using the compiler option "`-Werror`" and + then "`-Wno-error`" later on the command line. -- Fixed a regression in 3.7.2 when using the compiler option “-Werror” and then - “-Wno-error” later on the command line. +== Ccache 3.7.2 -ccache 3.7.2 ------------- Release date: 2019-07-19 -Bug fixes -~~~~~~~~~ -- The compiler option `-gdwarf*` no longer forces “run_second_cpp = true”. +=== Bug fixes + +- The compiler option `-gdwarf*` no longer forces "`run_second_cpp = true`". - Added verification that the value passed to the `-o/--set-config` option is valid. @@ -798,11 +1140,10 @@ - Unknown manifest versions are now handled gracefully in `--dump-manifest`. -- Fixed `make check` with “funny” locales. +- Fixed `make check` with "`funny`" locales. -Documentation -~~~~~~~~~~~~~ +=== Documentation - Added a hint about not running `autogen.sh` when building from a release archive. @@ -810,28 +1151,28 @@ - Mention that `xsltproc` is needed when building from the source repository. -ccache 3.7.1 ------------- +== Ccache 3.7.1 + Release date: 2019-05-01 -Changes -~~~~~~~ + +=== Changes - Fixed a problem when using the compiler option `-MF /dev/null`. - Long commandlines are now handled gracefully on Windows by using the `@file` syntax to avoid hitting the commandline size limit. -- Fixed complaint from GCC 9’s `-Werror=format-overflow` when compiling ccache +- Fixed complaint from GCC 9's `-Werror=format-overflow` when compiling ccache itself. -ccache 3.7 ----------- +== Ccache 3.7 + Release date: 2019-04-23 -Changes -~~~~~~~ + +=== Changes - Fixed crash when the debug mode is enabled and the output file is in a non-writable directory, e.g. when the output file is `/dev/null`. @@ -875,12 +1216,12 @@ machine-parsable (tab-separated) format. - ccache no longer creates a missing output directory, thus mimicking the - compiler behavior for `-o out/obj.o` when “out” doesn’t exist. + compiler behavior for `-o out/obj.o` when `out` doesn't exist. -- `-fdebug-prefix-map=ARG`, `-ffile-prefix-map=ARG` and - `-fmacro-prefix-map=ARG` are now included in the hash, but only the part - before “ARG”. This fixes a bug where compiler feature detection of said flags - would not work correctly with ccache. +- `-fdebug-prefix-map=ARG`, `-ffile-prefix-map=ARG` and `-fmacro-prefix-map=ARG` + are now included in the hash, but only the part before "`ARG`". This fixes a + bug where compiler feature detection of said flags would not work correctly + with ccache. - Bail out on too hard compiler option `-gtoggle`. @@ -903,19 +1244,19 @@ involved. -ccache 3.6 ----------- +== Ccache 3.6 + Release date: 2019-01-14 -Changes -~~~~~~~ -- ccache now has an opt-in “depend mode”. When enabled, ccache never executes +=== Changes + +- ccache now has an opt-in "`depend mode`". When enabled, ccache never executes the preprocessor, which results in much lower cache miss overhead at the expense of a lower potential cache hit rate. The depend mode is only possible to use when the compiler option `-MD` or `-MMD` is used. -- Added support for GCC’s `-ffile-prefix-map` option. The `-fmacro-prefix-map` +- Added support for GCC's `-ffile-prefix-map` option. The `-fmacro-prefix-map` option is now also skipped from the hash. - Added support for multiple `-fsanitize-blacklist` arguments. @@ -927,7 +1268,7 @@ - Fixed a problem due to Clang overwriting the output file when compiling an assembler file. -- Clarified the manual to explain the reasoning behind the “file_macro” +- Clarified the manual to explain the reasoning behind the "`file_macro`" sloppiness setting in a better way. - ccache now handles several levels of nonexistent directories when rewriting @@ -936,27 +1277,27 @@ - A new sloppiness setting `clang_index_store` makes ccache skip the Clang compiler option `-index-store-path` and its argument when computing the manifest hash. This is useful if you use Xcode, which uses an index store - path derived from the local project path. Note that the index store won’t be + path derived from the local project path. Note that the index store won't be updated correctly on cache hits if you enable this option. - Rename sloppiness `no_system_headers` to `system_headers` for consistency with other options. `no_system_headers` can still be used as an (undocumented) alias. -- The GCC variables “DEPENDENCIES_OUTPUT” and “SUNPRO_DEPENDENCIES” are now +- The GCC variables "`DEPENDENCIES_OUTPUT`" and "`SUNPRO_DEPENDENCIES`" are now supported correctly. - The algorithm that scans for `__DATE_` and `__TIME__` tokens in the hashed - source code now doesn’t produce false positives for tokens where `__DATE__` + source code now doesn't produce false positives for tokens where `__DATE__` or `__TIME__` is a substring. -ccache 3.5.1 ------------- +== Ccache 3.5.1 + Release date: 2019-01-02 -Changes -~~~~~~~ + +=== Changes - Added missing getopt_long.c source file to release archive. @@ -967,17 +1308,17 @@ - Improved development mode build flags. -ccache 3.5 ----------- +== Ccache 3.5 + Release date: 2018-10-15 -Changes -~~~~~~~ + +=== Changes - Added a boolean `debug` (`CCACHE_DEBUG`) configuration option. When enabled, - ccache will create per-object debug files that are helpful e.g. when - debugging unexpected cache misses. See also the new “Cache debugging” section - in the manual. + ccache will create per-object debug files that are helpful e.g. when debugging + unexpected cache misses. See also the new "`Cache debugging`" section in the + manual. - Renamed `CCACHE_CC` to `CCACHE_COMPILER` (keeping the former as a deprecated alias). @@ -993,36 +1334,36 @@ - Improved performance substantially when using `hash_dir = false` on platforms like macOS where `getcwd()` is slow. -- Added “stats updated” timestamp in `ccache -s` output. This can be useful if +- Added "`stats updated`" timestamp in `ccache -s` output. This can be useful if you wonder whether ccache actually was used for your last build. -- Renamed “stats zero time” to “stats zeroed” and documented it. The counter is - also now only present in `ccache -s` output when `ccache -z` actually has +- Renamed "`stats zero time`" to "`stats zeroed`" and documented it. The counter + is also now only present in `ccache -s` output when `ccache -z` actually has been called. - The content of the `-fsanitize-blacklist` file is now included in the hash, so updates to the file will now correctly result in separate cache entries. -- It’s now possible to opt out of building and installing man pages when +- It's now possible to opt out of building and installing man pages when running `make install` in the source repository. -- If the compiler type can’t be detected (e.g. if it is named `cc`), use safer - defaults that won’t trip up Clang. +- If the compiler type can't be detected (e.g. if it is named `cc`), use safer + defaults that won't trip up Clang. - Made the ccache test suite work on FreeBSD. - Added `file_stat_matches_ctime` option to disable ctime check if `file_stat_matches` is enabled. -- Made “./configure --without-bundled-zlib” do what’s intended. +- Made "`./configure --without-bundled-zlib`" do what's intended. + +== Ccache 3.4.3 -ccache 3.4.3 ------------ Release date: 2018-09-02 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a race condition when creating the initial config file in the cache directory. @@ -1036,12 +1377,12 @@ - Upgraded bundled zlib to version 1.2.11. -ccache 3.4.2 ------------- +== Ccache 3.4.2 + Release date: 2018-03-25 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - The cleanup algorithm has been fixed to not misbehave when files are removed by another process while the cleanup process is running. Previously, too many @@ -1049,42 +1390,42 @@ triggered at the same time, in extreme cases trimming the cache to a much smaller size than the configured limits. -- Correctly hash preprocessed headers located in a “.gch directory”. +- Correctly hash preprocessed headers located in a "`.gch directory`". Previously, ccache would not pick up changes to such precompiled headers, risking false positive cache hits. - Fixed build failure when using the bundled zlib sources. - ccache 3.3.5 added a workaround for not triggering Clang errors when a - precompiled header’s dependency has an updated timestamp (but identical + precompiled header's dependency has an updated timestamp (but identical content). That workaround is now only applied when the compiler is Clang. - Made it possible to perform out-of-source builds in dev mode again. -ccache 3.4.1 ------------- +== Ccache 3.4.1 + Release date: 2018-02-11 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed printing of version number in `ccache --version`. -ccache 3.4 ----------- +== Ccache 3.4 + Release date: 2018-02-11 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +=== New features and enhancements - The compiler option form `--sysroot arg` is now handled like the documented `--sysroot=arg` form. - Added support for caching `.su` files generated by GCC flag `-fstack-usage`. -- ccache should now work with distcc’s “pump” wrapper. +- ccache should now work with distcc's "`pump`" wrapper. - The optional unifier is no longer disabled when the direct mode is enabled. @@ -1093,7 +1434,7 @@ - Boolean environment variable settings no longer accept the following (case-insensitive) values: `0`, `false`, `disable` and `no`. All other values - are accepted and taken to mean “true”. This is to stop users from setting + are accepted and taken to mean "`true`". This is to stop users from setting e.g. `CCACHE_DISABLE=0` and then expect the cache to be used. - Improved support for `run_second_cpp = false`: If combined with passing @@ -1102,7 +1443,7 @@ - An implicit `-MQ` is now passed to the preprocessor only if the object file extension is non-standard. This should make it easier to use EDG-based - compilers (e.g. GHS) which don’t understand `-MQ`. + compilers (e.g. GHS) which don't understand `-MQ`. - ccache now treats an unreadable configuration file just like a missing configuration file. @@ -1112,8 +1453,7 @@ - Documented caveats related to colored warnings from compilers. -Bug fixes -~~~~~~~~~ +=== Bug fixes - File size and number counters are now updated correctly when files are overwritten in the cache, e.g. when using `CCACHE_RECACHE`. @@ -1123,39 +1463,35 @@ - Fixed how the NVCC options `-optf` and `-odir` are handled. -ccache 3.3.6 ------------- +== Ccache 3.3.6 + Release date: 2018-01-28 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Improved instructions on how to get cache hits between different working directories. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed regression in ccache 3.3.5 related to the `UNCACHED_ERR_FD` feature. -ccache 3.3.5 ------------- +== Ccache 3.3.5 + Release date: 2018-01-13 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Documented how automatic cache cleanup works. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a regression where the original order of debug options could be lost. - This reverts the “Improved parsing of `-g*` options” feature in ccache 3.3. + This reverts the "`Improved parsing of `-g*` options`" feature in ccache 3.3. - Multiple `-fdebug-prefix-map` options should now be handled correctly. @@ -1173,9 +1509,9 @@ - `ccache -c/--cleanup` now works like documented: it just recalculates size counters and trims the cache to not exceed the max size and file number - limits. Previously, the forced cleanup took “limit_multiple” into account, so - that `ccache -c/--cleanup` by default would trim the cache to 80% of the max - limit. + limits. Previously, the forced cleanup took "`limit_multiple`" into account, + so that `ccache -c/--cleanup` by default would trim the cache to 80% of the + max limit. - ccache no longer ignores linker arguments for Clang since Clang warns about them. @@ -1187,46 +1523,44 @@ `-remap` or `-trigraphs` option in preprocessor mode. -ccache 3.3.4 ------------- +== Ccache 3.3.4 + Release date: 2017-02-17 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Documented the different cache statistics counters. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a regression in ccache 3.3 related to potentially bad content of dependency files when compiling identical source code but with different source paths. This was only partially fixed in 3.3.2 and reverts the new - “Names of included files are no longer included in the hash of the compiler’s - preprocessed output” feature in 3.3. + "`Names of included files are no longer included in the hash of the compiler's + preprocessed output`" feature in 3.3. - Corrected statistics counter for `-optf`/`--options-file` failure. - Fixed undefined behavior warnings in ccache found by `-fsanitize=undefined`. -ccache 3.3.3 ------------- +== Ccache 3.3.3 + Release date: 2016-10-26 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - ccache now detects usage of `.incbin` assembler directives in the source code and avoids caching such compilations. -ccache 3.3.2 ------------- +== Ccache 3.3.2 + Release date: 2016-09-28 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a regression in ccache 3.3 related to potentially bad content of dependency files when compiling identical source code but with different @@ -1237,14 +1571,14 @@ resulting in missing dependency files from direct mode cache hits. -ccache 3.3.1 ------------- +== Ccache 3.3.1 + Release date: 2016-09-07 -Bug fixes -~~~~~~~~~ -- Fixed a problem in the “multiple `-arch` options” support introduced in 3.3. +=== Bug fixes + +- Fixed a problem in the "`multiple `-arch` options`" support introduced in 3.3. When using the direct mode (the default), different combinations of `-arch` options were not detected properly. @@ -1253,22 +1587,20 @@ (`CCACHE_CPP2`) is enabled. -ccache 3.3 ----------- +== Ccache 3.3 + Release date: 2016-08-27 -Notes -~~~~~ +=== Notes - A C99-compatible compiler is now required to build ccache. -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - The configuration option `run_second_cpp` (`CCACHE_CPP2`) now defaults to - true. This improves ccache’s out-of-the-box experience for compilers that - can’t compile their own preprocessed output with the same outcome as if they + true. This improves ccache's out-of-the-box experience for compilers that + can't compile their own preprocessed output with the same outcome as if they compiled the real source code directly, e.g. newer versions of GCC and Clang. - The configuration option `hash_dir` (`CCACHE_HASHDIR`) now defaults to true. @@ -1292,22 +1624,22 @@ - Added a new statistics counter that tracks the number of performed cleanups due to the cache size being over the limit. The value is shown in the output - of “ccache -s”. + of "`ccache -s`". - Added support for relocating debug info directory using `-fdebug-prefix-map`. This allows for cache hits even when `hash_dir` is used in combination with `base_dir`. -- Added a new “cache hit rate” field to the output of “ccache -s”. +- Added a new "`cache hit rate`" field to the output of "`ccache -s`". -- Added support for caching compilation of assembler code produced by e.g. “gcc - -S file.c”. +- Added support for caching compilation of assembler code produced by e.g. "`gcc + -S file.c`". - Added support for cuda including the -optf/--options-file option. - Added support for Fortran 77. -- Added support for multiple `-arch` options to produce “fat binaries”. +- Added support for multiple `-arch` options to produce "`fat binaries`". - Multiple identical `-arch` arguments are now handled without bailing. @@ -1329,7 +1661,7 @@ - ccache now understands the undocumented `-coverage` (only one dash) GCC option. -- Names of included files are no longer included in the hash of the compiler’s +- Names of included files are no longer included in the hash of the compiler's preprocessed output. This leads to more potential cache hits when not using the direct mode. @@ -1337,8 +1669,7 @@ slightly. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Bail out on too hard compiler option `-P`. @@ -1347,24 +1678,24 @@ - Fixed build and test for MinGW32 and Windows. -ccache 3.2.9 ------------- +== Ccache 3.2.9 + Release date: 2016-09-28 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a regression in ccache 3.2.8: ccache could get confused when using the compiler option `-Wp,` to pass multiple options to the preprocessor, resulting in missing dependency files from direct mode cache hits. -ccache 3.2.8 ------------- +== Ccache 3.2.8 + Release date: 2016-09-07 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed an issue when compiler option `-Wp,-MT,path` is used instead of `-MT path` (and similar for `-MF`, `-MP` and `-MQ`) and `run_second_cpp` @@ -1374,86 +1705,83 @@ option. -ccache 3.2.7 ------------- +== Ccache 3.2.7 + Release date: 2016-07-20 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a bug which could lead to false cache hits for compiler command lines with a missing argument to an option that takes an argument. -- ccache now knows how to work around a glitch in the output of GCC 6’s +- ccache now knows how to work around a glitch in the output of GCC 6's preprocessor. -ccache 3.2.6 ------------- +== Ccache 3.2.6 + Release date: 2016-07-12 -Bug fixes -~~~~~~~~~ -- Fixed build problem on QNX, which lacks “SA_RESTART”. +=== Bug fixes + +- Fixed build problem on QNX, which lacks "`SA_RESTART`". - Bail out on compiler option `-fstack-usage` since it creates a `.su` file - which ccache currently doesn’t handle. + which ccache currently doesn't handle. - Fixed a bug where (due to ccache rewriting paths) the compiler could choose incorrect include files if `CCACHE_BASEDIR` is used and the source file path is absolute and is a symlink. -ccache 3.2.5 ------------- +== Ccache 3.2.5 + Release date: 2016-04-17 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Only pass Clang-specific `-stdlib=` to the preprocessor. - Improved handling of stale NFS handles. -- Made it harder to misinterpret documentation of boolean environment settings’ +- Made it harder to misinterpret documentation of boolean environment settings' semantics. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Include m4 files used by configure.ac in the source dist archives. -- Corrected “Performance” section in the manual regarding `__DATE_`, `__TIME__` - and `__FILE__` macros. +- Corrected "`Performance`" section in the manual regarding `__DATE_`, + `__TIME__` and `__FILE__` macros. - Fixed build on Solaris 10+ and AIX 7. - Fixed failure to create directories on QNX. -- Don’t (try to) update manifest file in “read-only” and “read-only direct” +- Don't (try to) update manifest file in "`read-only`" and "`read-only direct`" modes. -- Fixed a bug in caching of `stat` system calls in “file_stat_matches - sloppiness mode”. +- Fixed a bug in caching of `stat` system calls in "`file_stat_matches + sloppiness mode`". - Fixed bug in hashing of Clang plugins, leading to unnecessary cache misses. -- Fixed --print-config to show “pch_defines sloppiness”. +- Fixed --print-config to show "`pch_defines sloppiness`". -- The man page is now built when running “make install” from Git repository +- The man page is now built when running "`make install`" from Git repository sources. -ccache 3.2.4 ------------- +== Ccache 3.2.4 + Release date: 2015-10-08 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed build error related to zlib on systems with older make versions (regression in ccache 3.2.3). @@ -1472,19 +1800,17 @@ 64 GiB on 32-bit systems. -ccache 3.2.3 ------------- +== Ccache 3.2.3 + Release date: 2015-08-16 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for compiler option `-gsplit-dwarf`. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Support external zlib in nonstandard directory. @@ -1494,34 +1820,32 @@ - Bail out on compiler option `--save-temps` in addition to `-save-temps`. -- Only log “Disabling direct mode” once when failing to read potential include +- Only log "`Disabling direct mode`" once when failing to read potential include files. -ccache 3.2.2 ------------- +== Ccache 3.2.2 + Release date: 2015-05-10 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for `CCACHE_COMPILERCHECK=string:`. This is a faster - alternative to `CCACHE_COMPILERCHECK=` if the command’s output can + alternative to `CCACHE_COMPILERCHECK=` if the command's output can be precalculated by the build system. - Add support for caching code coverage results (compiling for gcov). -Bug fixes -~~~~~~~~~ +=== Bug fixes - Made hash of cached result created with and without `CCACHE_CPP2` different. This makes it possible to rebuild with `CCACHE_CPP2` set without having to clear the cache to get new results. -- Don’t try to reset a nonexistent stats file. This avoids “No such file or - directory” messages in the ccache log when the cache directory doesn’t exist. +- Don't try to reset a nonexistent stats file. This avoids "`No such file or + directory`" messages in the ccache log when the cache directory doesn't exist. - Fixed a bug where ccache deleted Clang diagnostics after compiler failures. @@ -1543,13 +1867,12 @@ - Fixed build error when compiling ccache with recent Clang versions. -ccache 3.2.1 ------------- +== Ccache 3.2.1 + Release date: 2014-12-10 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed regression in temporary file handling, which lead to incorrect permissions for stats, manifest and ccache.conf files in the cache. @@ -1567,13 +1890,12 @@ options. -ccache 3.2 ----------- +== Ccache 3.2 + Release date: 2014-11-17 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for configuring ccache via one or several configuration files instead of via environment variables. Environment variables still have @@ -1591,7 +1913,7 @@ - Added support for several binaries (separated by space) in `CCACHE_PREFIX`. - The `-c` option is no longer passed to the preprocessor. This fixes problems - with Clang and Solaris’s C++ compiler. + with Clang and Solaris's C++ compiler. - ccache no longer passes preprocessor options like `-D` and `-I` to the compiler when compiling preprocessed output. This fixes warnings emitted by @@ -1600,7 +1922,7 @@ - Compiler options `-fprofile-generate`, `-fprofile-arcs`, `-fprofile-use` and `-fbranch-probabilities` are now handled without bailing. -- Added support for Clang’s `--serialize-diagnostic` option, storing the +- Added support for Clang's `--serialize-diagnostic` option, storing the diagnostic file (`.dia`) in the cache. - Added support for precompiled headers when using Clang. @@ -1615,19 +1937,19 @@ the other way around. This is needed to support compiler options like `-fprofile-arcs` and `--serialize-diagnostics`. -- ccache now checks that included files’ ctimes aren’t too new. This check can - be turned off by adding `include_file_ctime` to the “ccache sloppiness” +- ccache now checks that included files' ctimes aren't too new. This check can + be turned off by adding `include_file_ctime` to the "`ccache sloppiness`" setting. - Added possibility to get cache hits based on filename, size, mtime and ctime only. On other words, source code files are not even read, only stat-ed. This - operation mode is opt-in by adding `file_stat_matches` to the “ccache - sloppiness” setting. + operation mode is opt-in by adding `file_stat_matches` to the "`ccache + sloppiness`" setting. - The filename part of options like `-Wp,-MDfilename` is no longer included in - the hash since the filename doesn’t have any bearing on the result. + the hash since the filename doesn't have any bearing on the result. -- Added a “read-only direct” configuration setting, which is like the ordinary +- Added a "`read-only direct`" configuration setting, which is like the ordinary read-only setting except that ccache will only try to retrieve results from the cache using the direct mode, not the preprocessor mode. @@ -1642,7 +1964,7 @@ - Added support for `@file` and `-@file` arguments (reading options from a file). -- `-Wl,` options are no longer included in the hash since they don’t affect +- `-Wl,` options are no longer included in the hash since they don't affect compilation. - Bail out on too hard compiler option `-Wp,-P`. @@ -1666,8 +1988,7 @@ - Various other improvements of the test suite. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Any previous `.stderr` is now removed from the cache when recaching. @@ -1678,26 +1999,24 @@ - Fixed test suite failures when `CC` is a ccache-wrapped compiler. -ccache 3.1.12 -------------- +== Ccache 3.1.12 + Release date: 2016-07-12 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a bug where (due to ccache rewriting paths) the compiler could choose incorrect include files if `CCACHE_BASEDIR` is used and the source file path is absolute and is a symlink. -ccache 3.1.11 -------------- +== Ccache 3.1.11 + Release date: 2015-03-07 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed bug which could result in false cache hits when source code contains `'"'` followed by `" /*"` or `" //"` (with variations). @@ -1706,17 +2025,16 @@ This makes it possible to rebuild with `CCACHE_CPP2` set without having to clear the cache to get new results. -- Don’t try to reset a nonexistent stats file. This avoids “No such file or - directory” messages in the ccache log when the cache directory doesn’t exist. +- Don't try to reset a nonexistent stats file. This avoids "`No such file or + directory`" messages in the ccache log when the cache directory doesn't exist. -ccache 3.1.10 -------------- +== Ccache 3.1.10 + Release date: 2014-10-19 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for the `-Xclang` compiler option. @@ -1731,21 +2049,20 @@ `CCACHE_BASEDIR` to reuse results across different directories.) - Added note in documentation that `--ccache-skip` currently does not mean - “don’t hash the following option”. + "`don't hash the following option`". - To enable support for precompiled headers (PCH), `CCACHE_SLOPPINESS` now also needs to include the new `pch_defines` sloppiness. This is because ccache - can’t detect changes in the source code when only defined macros have been + can't detect changes in the source code when only defined macros have been changed. - Stale files in the internal temporary directory (`/tmp`) are now cleaned up if they are older than one hour. -Bug fixes -~~~~~~~~~ +=== Bug fixes -- Fixed path canonicalization in `make_relative_path()` when path doesn’t +- Fixed path canonicalization in `make_relative_path()` when path doesn't exist. - Fixed bug in `common_dir_prefix_length()`. This corrects the `CCACHE_BASEDIR` @@ -1760,13 +2077,12 @@ - Fixed problem with logging of current working directory. -ccache 3.1.9 ------------- +== Ccache 3.1.9 + Release date: 2013-01-06 -Bug fixes -~~~~~~~~~ +=== Bug fixes - The EAGAIN signal is now handled correctly when emitting cached stderr output. This fixes a problem triggered by large error outputs from the @@ -1774,7 +2090,7 @@ - Subdirectories in the cache are no longer created in read-only mode. -- Fixed so that ccache’s log file descriptor is not made available to the +- Fixed so that ccache's log file descriptor is not made available to the compiler. - Improved error reporting when failing to create temporary stdout/stderr files @@ -1783,19 +2099,17 @@ - Disappearing temporary stdout/stderr files are now handled gracefully. -Other -~~~~~ +=== Other - Fixed test suite to work on ecryptfs. -ccache 3.1.8 ------------- +== Ccache 3.1.8 + Release date: 2012-08-11 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Made paths to dependency files relative in order to increase cache hits. @@ -1805,8 +2119,7 @@ - Clang plugins are now hashed to catch plugin upgrades. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed crash when the current working directory has been removed. @@ -1819,28 +2132,26 @@ base directory. -Other -~~~~~ +=== Other - Made git version macro work when compiling outside of the source directory. - Fixed `static_assert` macro definition clash with GCC 4.7. -ccache 3.1.7 ------------- +== Ccache 3.1.7 + Release date: 2012-01-08 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Non-writable `CCACHE_DIR` is now handled gracefully when `CCACHE_READONLY` is set. - Made failure to create files (typically due to bad directory permissions) in the cache directory fatal. Previously, such failures were silently and - erroneously flagged as “compiler produced stdout”. + erroneously flagged as "`compiler produced stdout`". - Both the `-specs=file` and `--specs=file` forms are now recognized. @@ -1859,8 +2170,7 @@ versions.) -Other -~~~~~ +=== Other - Corrected license header for `mdfour.c`. @@ -1868,34 +2178,31 @@ -ccache 3.1.6 ------------- +== Ccache 3.1.6 + Release date: 2011-08-21 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Rewrite argument to `--sysroot` if `CCACHE_BASEDIR` is used. -Bug fixes -~~~~~~~~~ +=== Bug fixes + +- Don't crash if `getcwd()` fails. -- Don’t crash if `getcwd()` fails. +- Fixed alignment of "`called for preprocessing`" counter. -- Fixed alignment of “called for preprocessing” counter. +== Ccache 3.1.5 -ccache 3.1.5 ------------- Release date: 2011-05-29 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements -- Added a new statistics counter named “called for preprocessing”. +- Added a new statistics counter named "`called for preprocessing`". - The original command line is now logged to the file specified with `CCACHE_LOGFILE`. @@ -1908,8 +2215,7 @@ - Improved order of statistics counters in `ccache -s` output. -Bug fixes -~~~~~~~~~ +=== Bug fixes - The `-MF`/`-MT`/`-MQ` options with concatenated argument are now handled correctly when they are last on the command line. @@ -1919,53 +2225,49 @@ - Fixed a minor memory leak. -- Systems that lack (and don’t need to be linked with) libm are now supported. +- Systems that lack (and don't need to be linked with) libm are now supported. + +== Ccache 3.1.4 -ccache 3.1.4 ------------- Release date: 2011-01-09 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Made a work-around for a bug in `gzputc()` in zlib 1.2.5. -- Corrupt manifest files are now removed so that they won’t block direct mode +- Corrupt manifest files are now removed so that they won't block direct mode hits. -- ccache now copes with file systems that don’t know about symbolic links. +- ccache now copes with file systems that don't know about symbolic links. -- The file handle in now correctly closed on write error when trying to create +- The file handle is now correctly closed on write error when trying to create a cache dir tag. -ccache 3.1.3 ------------- +== Ccache 3.1.3 + Release date: 2010-11-28 -Bug fixes -~~~~~~~~~ +=== Bug fixes - The -MFarg, -MTarg and -MQarg compiler options (i.e, without space between option and argument) are now handled correctly. -Other -~~~~~ +=== Other - Portability fixes for HP-UX 11.00 and other esoteric systems. -ccache 3.1.2 ------------- +== Ccache 3.1.2 + Release date: 2010-11-21 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Bail out on too hard compiler options `-fdump-*`. @@ -1975,24 +2277,22 @@ - Fixed issue when parsing precompiler output on AIX. -Other -~~~~~ +=== Other - Improved documentation on which information is included in the hash sum. -- Made the “too new header file” test case work on file systems with +- Made the "`too new header file`" test case work on file systems with unsynchronized clocks. - The test suite now also works on systems that lack a /dev/zero. -ccache 3.1.1 ------------- +== Ccache 3.1.1 + Release date: 2010-11-07 -Bug fixes -~~~~~~~~~ +=== Bug fixes - ccache now falls back to preprocessor mode when a non-regular include file (device, socket, etc) has been detected so that potential hanging due to @@ -2007,27 +2307,25 @@ - Fixed configure detection of ar. - ccache development version (set by dev.mk) now works with gits whose - `describe` command doesn’t understand `--dirty`. + `describe` command doesn't understand `--dirty`. -Other -~~~~~ +=== Other - Minor debug log message improvements. -ccache 3.1 ----------- +== Ccache 3.1 + Release date: 2010-09-16 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for hashing the output of a custom command (e.g. `%compiler% --version`) to identify the compiler instead of stat-ing or hashing the compiler binary. This can improve robustness when the compiler (as seen by - ccache) actually isn’t the real compiler but another compiler wrapper. + ccache) actually isn't the real compiler but another compiler wrapper. - Added support for caching compilations that use precompiled headers. (See the manual for important instructions regarding this.) @@ -2046,9 +2344,9 @@ - Reading and writing of statistics counters has been made forward-compatible (unknown counters are retained). -- Files are now read without using `mmap()`. This has two benefits: it’s more +- Files are now read without using `mmap()`. This has two benefits: it's more robust against file changes during reading and it improves performance on - poor systems where `mmap()` doesn’t use the disk cache. + poor systems where `mmap()` doesn't use the disk cache. - Added `.cp` and `.CP` as known C++ suffixes. @@ -2058,8 +2356,7 @@ statistics when using the Darwin linker.) -Bug fixes -~~~~~~~~~ +=== Bug fixes - Non-fatal error messages are now never printed to stderr but logged instead. @@ -2070,8 +2367,7 @@ - EINTR is now handled correctly. -Other -~~~~~ +=== Other - Work on porting ccache to win32 (native), mostly done by Ramiro Polla. The port is not yet finished, but will hopefully be complete in some subsequent @@ -2097,22 +2393,21 @@ - New `HACKING.txt` file with some notes about ccache code conventions. -ccache 3.0.1 ------------- +== Ccache 3.0.1 + Release date: 2010-07-15 -Bug fixes -~~~~~~~~~ +=== Bug fixes -- The statistics counter “called for link” is now correctly updated when +- The statistics counter "`called for link`" is now correctly updated when linking with a single object file. - Fixed a problem with out-of-source builds. -ccache 3.0 ----------- +== Ccache 3.0 + Release date: 2010-06-20 @@ -2123,19 +2418,17 @@ or later. -Upgrade notes -~~~~~~~~~~~~~ +=== Upgrade notes -- The way the hashes are calculated has changed, so you won’t get cache hits +- The way the hashes are calculated has changed, so you won't get cache hits for compilation results stored by older ccache versions. Because of this, you might as well clear the old cache directory with `ccache --clear` if you want, unless you plan to keep using an older ccache version. -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements -- ccache now has a “direct mode” where it computes a hash of the source code +- ccache now has a "`direct mode`" where it computes a hash of the source code (including all included files) and compiler options without running the preprocessor. By not running the preprocessor, CPU usage is reduced; the speed is somewhere between 1 and 5 times that of ccache running in @@ -2153,19 +2446,19 @@ - Object files are now optionally stored compressed in the cache. The runtime cost is negligible, and more files will fit in the ccache directory and in the disk cache. Set `CCACHE_COMPRESS` to enable object file compression. Note - that you can’t use compression in combination with the hard link feature. + that you can't use compression in combination with the hard link feature. - A `CCACHE_COMPILERCHECK` option has been added. This option tells ccache what compiler-identifying information to hash to ensure that results retrieved - from the cache are accurate. Possible values are: none (don’t hash anything), - mtime (hash the compiler’s mtime and size) and content (hash the content of + from the cache are accurate. Possible values are: none (don't hash anything), + mtime (hash the compiler's mtime and size) and content (hash the content of the compiler binary). The default is mtime. - It is now possible to specify extra files whose contents should be included in the hash sum by setting the `CCACHE_EXTRAFILES` option. - Added support for Objective-C and Objective-C\+\+. The statistics counter - “not a C/C++ file” has been renamed to “unsupported source language”. + "`not a C/C++ file`" has been renamed to "`unsupported source language`". - Added support for the `-x` compiler option. @@ -2188,13 +2481,13 @@ - Temporary files that later will be moved into the cache are now created in the cache directory they will end up in. This makes ccache more friendly to - Linux’s directory layout. + Linux's directory layout. - Improved the test suite and added tests for most of the new functionality. - It’s now also possible to specify a subset of tests to run. + It's now also possible to specify a subset of tests to run. - Standard error output from the compiler is now only stored in the cache if - it’s non-empty. + it's non-empty. - If the compiler produces no object file or an empty object file, but gives a zero exit status (could be due to a file system problem, a buggy program @@ -2202,7 +2495,7 @@ - Added `installcheck` and `distcheck` make targets. -- Clarified cache size limit options’ and cleanup semantics. +- Clarified cache size limit options' and cleanup semantics. - Improved display of cache max size values. @@ -2211,8 +2504,7 @@ `-iwithprefixbefore`, `-nostdinc`, `-nostdinc++` and `-U`. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Various portability improvements. @@ -2231,7 +2523,7 @@ `-save-temps`. Also bail out on `@file` style options. - Errors when using multiple `-arch` compiler options are now noted as - “unsupported compiler option”. + "`unsupported compiler option`". - `-MD`/`-MMD` options without `-MT`/`-MF` are now handled correctly. diff -Nru ccache-4.2.1/dockerfiles/alpine-3.12/Dockerfile ccache-4.5.1/dockerfiles/alpine-3.12/Dockerfile --- ccache-4.2.1/dockerfiles/alpine-3.12/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/alpine-3.12/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -FROM alpine:3.12 - -RUN apk add --no-cache \ - bash \ - ccache \ - clang \ - cmake \ - elfutils \ - g++ \ - gcc \ - libc-dev \ - make \ - perl \ - zstd-dev - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff -Nru ccache-4.2.1/dockerfiles/alpine-3.14/Dockerfile ccache-4.5.1/dockerfiles/alpine-3.14/Dockerfile --- ccache-4.2.1/dockerfiles/alpine-3.14/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/dockerfiles/alpine-3.14/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,20 @@ +FROM alpine:3.14 + +RUN apk add --no-cache \ + bash \ + ccache \ + clang \ + cmake \ + elfutils \ + g++ \ + gcc \ + hiredis-dev \ + libc-dev \ + make \ + perl \ + python3 \ + redis \ + zstd-dev + +# Redirect all compilers to ccache. +RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff -Nru ccache-4.2.1/dockerfiles/alpine-3.4/Dockerfile ccache-4.5.1/dockerfiles/alpine-3.4/Dockerfile --- ccache-4.2.1/dockerfiles/alpine-3.4/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/alpine-3.4/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -# Released 2016, this is the first release to contain cmake >= 3.4.3 - -FROM alpine:3.4 - -RUN apk add --no-cache \ - bash \ - ccache \ - cmake \ - g++ \ - gcc \ - libc-dev \ - make \ - perl - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff -Nru ccache-4.2.1/dockerfiles/alpine-3.8/Dockerfile ccache-4.5.1/dockerfiles/alpine-3.8/Dockerfile --- ccache-4.2.1/dockerfiles/alpine-3.8/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/dockerfiles/alpine-3.8/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,20 @@ +FROM alpine:3.8 + +RUN apk add --no-cache \ + bash \ + ccache \ + clang \ + cmake \ + elfutils \ + g++ \ + gcc \ + hiredis-dev \ + libc-dev \ + make \ + perl \ + python3 \ + redis \ + zstd-dev + +# Redirect all compilers to ccache. +RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff -Nru ccache-4.2.1/dockerfiles/centos-7/Dockerfile ccache-4.5.1/dockerfiles/centos-7/Dockerfile --- ccache-4.2.1/dockerfiles/centos-7/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/centos-7/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -1,21 +1,24 @@ FROM centos:7 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \ + && yum install -y centos-release-scl \ && yum install -y \ - asciidoc \ + asciidoctor \ autoconf \ bash \ ccache \ clang \ cmake3 \ + devtoolset-8 \ elfutils \ gcc \ gcc-c++ \ libzstd-devel \ make \ -# Remove superfluous dependencies brought in by asciidoc: - && rpm -e --nodeps graphviz \ + python3 \ && yum autoremove -y \ && yum clean all \ && cp /usr/bin/cmake3 /usr/bin/cmake \ && cp /usr/bin/ctest3 /usr/bin/ctest + +ENTRYPOINT ["scl", "enable", "devtoolset-8", "--"] diff -Nru ccache-4.2.1/dockerfiles/centos-8/Dockerfile ccache-4.5.1/dockerfiles/centos-8/Dockerfile --- ccache-4.2.1/dockerfiles/centos-8/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/centos-8/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -1,8 +1,9 @@ FROM centos:8 +# also run update due to https://bugs.centos.org/view.php?id=18212 RUN dnf install -y epel-release \ + && dnf update -y \ && dnf install -y \ - asciidoc \ autoconf \ bash \ ccache \ @@ -12,6 +13,9 @@ elfutils \ gcc \ gcc-c++ \ + hiredis-devel \ libzstd-devel \ make \ + python3 \ + redis \ && dnf clean all diff -Nru ccache-4.2.1/dockerfiles/debian-10/Dockerfile ccache-4.5.1/dockerfiles/debian-10/Dockerfile --- ccache-4.2.1/dockerfiles/debian-10/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/debian-10/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -9,8 +9,11 @@ cmake \ elfutils \ gcc-multilib \ + libhiredis-dev \ libzstd-dev \ - xsltproc \ + python3 \ + redis-server \ + redis-tools \ && rm -rf /var/lib/apt/lists/* # Redirect all compilers to ccache. diff -Nru ccache-4.2.1/dockerfiles/debian-11/Dockerfile ccache-4.5.1/dockerfiles/debian-11/Dockerfile --- ccache-4.2.1/dockerfiles/debian-11/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/dockerfiles/debian-11/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,20 @@ +FROM debian:11 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ccache \ + clang \ + cmake \ + elfutils \ + gcc-multilib \ + libhiredis-dev \ + libzstd-dev \ + python3 \ + redis-server \ + redis-tools \ + && rm -rf /var/lib/apt/lists/* + +# Redirect all compilers to ccache. +RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff -Nru ccache-4.2.1/dockerfiles/debian-9/Dockerfile ccache-4.5.1/dockerfiles/debian-9/Dockerfile --- ccache-4.2.1/dockerfiles/debian-9/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/debian-9/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -FROM debian:9 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - bash \ - build-essential \ - ccache \ - clang \ - cmake \ - elfutils \ - gcc-multilib \ - libzstd-dev \ - xsltproc \ - && rm -rf /var/lib/apt/lists/* - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff -Nru ccache-4.2.1/dockerfiles/fedora-32/Dockerfile ccache-4.5.1/dockerfiles/fedora-32/Dockerfile --- ccache-4.2.1/dockerfiles/fedora-32/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/fedora-32/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -11,6 +11,9 @@ findutils \ gcc \ gcc-c++ \ + hiredis-devel \ libzstd-devel \ make \ + python3 \ + redis \ && dnf clean all diff -Nru ccache-4.2.1/dockerfiles/README ccache-4.5.1/dockerfiles/README --- ccache-4.2.1/dockerfiles/README 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/README 2021-11-17 19:31:58.000000000 +0000 @@ -3,7 +3,7 @@ For instance, run something like this to build ccache in Ubuntu 20.04: - misc/build-in-docker ubuntu-20-focal + misc/build-in-docker ubuntu-20.04 The above command will first build the Ubuntu 20.04 Docker image if needed and finally build ccache and run the ccache test suite. diff -Nru ccache-4.2.1/dockerfiles/ubuntu-14.04/Dockerfile ccache-4.5.1/dockerfiles/ubuntu-14.04/Dockerfile --- ccache-4.2.1/dockerfiles/ubuntu-14.04/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/ubuntu-14.04/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,26 +0,0 @@ -FROM ubuntu:14.04 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - asciidoc \ - bash \ - build-essential \ - ccache \ - clang-3.4 \ - curl \ - docbook-xml \ - docbook-xsl \ - elfutils \ - g++-multilib \ - ninja-build \ - wget \ - xsltproc \ - && rm -rf /var/lib/apt/lists/* - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done - -# The distribution's CMake it too old (2.8.12.2). -RUN curl -sSL https://cmake.org/files/v3.5/cmake-3.5.2-Linux-x86_64.tar.gz | sudo tar -xzC /opt \ - && cp -a /opt/cmake-3.5.2-Linux-x86_64/bin /usr/local \ - && cp -a /opt/cmake-3.5.2-Linux-x86_64/share /usr/local diff -Nru ccache-4.2.1/dockerfiles/ubuntu-16.04/Dockerfile ccache-4.5.1/dockerfiles/ubuntu-16.04/Dockerfile --- ccache-4.2.1/dockerfiles/ubuntu-16.04/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/ubuntu-16.04/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,20 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - asciidoc \ - bash \ - build-essential \ - ccache \ - clang \ - cmake \ - docbook-xml \ - docbook-xsl \ - elfutils \ - gcc-multilib \ - libzstd1-dev \ - xsltproc \ - && rm -rf /var/lib/apt/lists/* - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff -Nru ccache-4.2.1/dockerfiles/ubuntu-18.04/Dockerfile ccache-4.5.1/dockerfiles/ubuntu-18.04/Dockerfile --- ccache-4.2.1/dockerfiles/ubuntu-18.04/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/ubuntu-18.04/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -2,7 +2,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ - asciidoc \ + asciidoctor \ bash \ build-essential \ ccache \ @@ -12,8 +12,11 @@ docbook-xsl \ elfutils \ gcc-multilib \ + libhiredis-dev \ libzstd-dev \ - xsltproc \ + python3 \ + redis-server \ + redis-tools \ && rm -rf /var/lib/apt/lists/* # Redirect all compilers to ccache. diff -Nru ccache-4.2.1/dockerfiles/ubuntu-20.04/Dockerfile ccache-4.5.1/dockerfiles/ubuntu-20.04/Dockerfile --- ccache-4.2.1/dockerfiles/ubuntu-20.04/Dockerfile 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/dockerfiles/ubuntu-20.04/Dockerfile 2021-11-17 19:31:58.000000000 +0000 @@ -3,7 +3,7 @@ # Non-interactive: do not set up timezone settings. RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ - asciidoc \ + asciidoctor \ bash \ build-essential \ ccache \ @@ -13,8 +13,11 @@ docbook-xsl \ elfutils \ gcc-multilib \ + libhiredis-dev \ libzstd-dev \ - xsltproc \ + python3 \ + redis-server \ + redis-tools \ && rm -rf /var/lib/apt/lists/* # Redirect all compilers to ccache. diff -Nru ccache-4.2.1/.github/workflows/build.yaml ccache-4.5.1/.github/workflows/build.yaml --- ccache-4.2.1/.github/workflows/build.yaml 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/.github/workflows/build.yaml 2021-11-17 19:31:58.000000000 +0000 @@ -22,14 +22,6 @@ fail-fast: false matrix: config: - - os: ubuntu-16.04 - compiler: gcc - version: "4.8" # results in 4.8.5 - - - os: ubuntu-16.04 - compiler: gcc - version: "5" - - os: ubuntu-18.04 compiler: gcc version: "6" @@ -50,11 +42,11 @@ compiler: gcc version: "10" - - os: ubuntu-16.04 + - os: ubuntu-18.04 compiler: clang version: "5.0" - - os: ubuntu-16.04 + - os: ubuntu-18.04 compiler: clang version: "6.0" @@ -74,28 +66,37 @@ compiler: clang version: "10" - - os: macOS-latest + - os: ubuntu-20.04 + compiler: clang + version: "11" + + - os: ubuntu-20.04 + compiler: clang + version: "12" + + - os: macOS-10.15 compiler: xcode version: "10.3" - - os: macOS-latest + - os: macOS-10.15 compiler: xcode version: "11.7" - - os: macOS-latest + - os: macOS-10.15 compiler: xcode - version: "12.2" + version: "12.4" steps: - name: Install dependencies run: | if [ "${{ runner.os }}" = "Linux" ]; then sudo apt-get update - # Install ld.gold (binutils) and ld.lld on different runs. - if [ "${{ matrix.config.os }}" = "ubuntu-16.04" ]; then - sudo apt-get install -y ninja-build elfutils libzstd1-dev binutils + packages="elfutils libhiredis-dev libzstd-dev ninja-build pkg-config python3 redis-server redis-tools" + # Install ld.gold (binutils) and ld.lld (lld) on different runs. + if [ "${{ matrix.config.os }}" = "ubuntu-18.04" ]; then + sudo apt-get install -y $packages binutils else - sudo apt-get install -y ninja-build elfutils libzstd-dev lld + sudo apt-get install -y $packages lld fi if [ "${{ matrix.config.compiler }}" = "gcc" ]; then @@ -107,11 +108,11 @@ echo "CC=clang-${{ matrix.config.version }}" >> $GITHUB_ENV echo "CXX=clang++-${{ matrix.config.version }}" >> $GITHUB_ENV - sudo apt install -y clang-${{ matrix.config.version }} g++-multilib + sudo apt install -y clang-${{ matrix.config.version }} g++-multilib lld-${{ matrix.config.version }} fi elif [ "${{ runner.os }}" = "macOS" ]; then HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 \ - brew install ninja + brew install ninja pkg-config hiredis redis if [ "${{ matrix.config.compiler }}" = "gcc" ]; then brew install gcc@${{ matrix.config.version }} @@ -151,15 +152,15 @@ fail-fast: false matrix: config: - - name: Linux GCC debug + C++14 + in source + tracing + - name: Linux GCC debug + in source + tracing os: ubuntu-18.04 CC: gcc CXX: g++ ENABLE_CACHE_CLEANUP_TESTS: 1 BUILDDIR: . CCACHE_LOC: . - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug -DENABLE_TRACING=1 -DCMAKE_CXX_STANDARD=14 - apt_get: elfutils libzstd-dev + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug -DENABLE_TRACING=1 + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev - name: Linux GCC 32-bit os: ubuntu-18.04 @@ -168,7 +169,7 @@ CFLAGS: -m32 -g -O2 CXXFLAGS: -m32 -g -O2 LDFLAGS: -m32 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON ENABLE_CACHE_CLEANUP_TESTS: 1 apt_get: elfutils gcc-multilib g++-multilib lib32stdc++-5-dev @@ -176,7 +177,7 @@ os: ubuntu-18.04 CC: gcc CXX: g++ - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON ENABLE_CACHE_CLEANUP_TESTS: 1 CUDA: 10.1.243-1 apt_get: elfutils libzstd-dev @@ -185,7 +186,7 @@ os: ubuntu-18.04 CC: i686-w64-mingw32-gcc-posix CXX: i686-w64-mingw32-g++-posix - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON RUN_TESTS: none apt_get: elfutils mingw-w64 @@ -194,7 +195,7 @@ CC: x86_64-w64-mingw32-gcc-posix CXX: x86_64-w64-mingw32-g++-posix ENABLE_CACHE_CLEANUP_TESTS: 1 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON RUN_TESTS: unittest-in-wine apt_get: elfutils mingw-w64 wine @@ -206,7 +207,7 @@ CXX: cl ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON TEST_CC: clang -target i686-pc-windows-msvc - name: Windows VS2019 64-bit @@ -217,7 +218,7 @@ CXX: cl ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON TEST_CC: clang -target x86_64-pc-windows-msvc - name: Clang address & UB sanitizer @@ -227,7 +228,7 @@ ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON ASAN_OPTIONS: detect_leaks=0 - apt_get: elfutils libzstd-dev + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev - name: Clang static analyzer os: ubuntu-20.04 @@ -236,7 +237,7 @@ ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_PREFIX: scan-build RUN_TESTS: none - apt_get: libzstd-dev + apt_get: libzstd-dev pkg-config libhiredis-dev - name: Linux binary os: ubuntu-20.04 @@ -244,34 +245,34 @@ CXX: g++ SPECIAL: build-and-verify-package CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Release - apt_get: elfutils libzstd-dev ninja-build + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build - name: Source package os: ubuntu-20.04 CC: gcc CXX: g++ SPECIAL: build-and-verify-source-package - apt_get: elfutils libzstd-dev ninja-build asciidoc xsltproc docbook-xml docbook-xsl + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build asciidoctor - name: HTML documentation os: ubuntu-18.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-html RUN_TESTS: none - apt_get: libzstd-dev asciidoc docbook-xml docbook-xsl + apt_get: libzstd-dev pkg-config libhiredis-dev asciidoctor - name: Manual page os: ubuntu-18.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-man-page RUN_TESTS: none - apt_get: libzstd-dev asciidoc xsltproc docbook-xml docbook-xsl + apt_get: libzstd-dev pkg-config libhiredis-dev asciidoctor - name: Clang-Tidy os: ubuntu-18.04 - CC: clang - CXX: clang++ + CC: clang-9 + CXX: clang++-9 RUN_TESTS: none - CMAKE_PARAMS: -DENABLE_CLANG_TIDY=ON - apt_get: libzstd-dev clang-tidy + CMAKE_PARAMS: -DENABLE_CLANG_TIDY=ON -DCLANGTIDY=/usr/bin/clang-tidy-9 + apt_get: libzstd-dev pkg-config libhiredis-dev clang-9 clang-tidy-9 steps: - name: Get source @@ -371,7 +372,10 @@ uses: actions/checkout@v2 - name: Install codespell - run: sudo apt-get update && sudo apt-get install codespell + run: | + sudo apt-get update + sudo apt-get install python3-pip + pip3 install codespell==2.1.0 - name: Run codespell - run: codespell -q 7 -S ".git,LICENSE.adoc,./src/third_party/*" -I misc/codespell-allowlist.txt + run: codespell -q 7 -S ".git,build*,./src/third_party/*" -I misc/codespell-allowlist.txt diff -Nru ccache-4.2.1/.github/workflows/codeql-analysis.yaml ccache-4.5.1/.github/workflows/codeql-analysis.yaml --- ccache-4.2.1/.github/workflows/codeql-analysis.yaml 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/.github/workflows/codeql-analysis.yaml 2021-11-17 19:31:58.000000000 +0000 @@ -31,7 +31,7 @@ fetch-depth: 2 - name: Install dependencies - run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev + run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev pkg-config libhiredis-dev - name: Initialize CodeQL uses: github/codeql-action/init@v1 diff -Nru ccache-4.2.1/.gitignore ccache-4.5.1/.gitignore --- ccache-4.2.1/.gitignore 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/.gitignore 2021-11-17 19:31:58.000000000 +0000 @@ -1,6 +1,13 @@ # Typical build directories /build*/ +# CLion +/.idea/ +/cmake-build-*/ + +# Downloaded tools +misc/.clang-format-exe + # Emacs save files *~ diff -Nru ccache-4.2.1/LICENSE.adoc ccache-4.5.1/LICENSE.adoc --- ccache-4.2.1/LICENSE.adoc 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/LICENSE.adoc 2021-11-17 19:31:58.000000000 +0000 @@ -1,12 +1,10 @@ -Ccache copyright and license -============================ += Ccache copyright and license -Overall license ---------------- +== Overall license The license for ccache as a whole is as follows: -------------------------------------------------------------------------------- +---- This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later @@ -19,14 +17,13 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -------------------------------------------------------------------------------- +---- The full license text can be found in GPL-3.0.txt and at https://www.gnu.org/licenses/gpl-3.0.html. -Copyright and authors ---------------------- +== Copyright and authors Ccache is a collective work with contributions from many people, listed in AUTHORS.adoc and at https://ccache.dev/credits.html. Subsequent additions by @@ -36,14 +33,13 @@ The copyright for ccache as a whole is as follows: -------------------------------------------------------------------------------- +---- Copyright (C) 2002-2007 Andrew Tridgell Copyright (C) 2009-2021 Joel Rosdahl and other contributors -------------------------------------------------------------------------------- +---- -Files derived from other sources --------------------------------- +== Files derived from other sources The ccache distribution contain some files from other sources and some have been modified for use in ccache. These files all carry attribution notices, and @@ -52,13 +48,12 @@ under less restrictive terms. -src/third_party/base32hex.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/base32hex.* This base32hex implementation comes from . -------------------------------------------------------------------------------- +---- (C) 2012 Peter Conrad This program is free software: you can redistribute it and/or modify @@ -72,21 +67,19 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -------------------------------------------------------------------------------- +---- -src/third_party/blake3/blake3_* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/blake3/blake3_* -This is a subset of https://github.com/BLAKE3-team/BLAKE3[BLAKE3] 0.3.7 with +This is a subset of https://github.com/BLAKE3-team/BLAKE3[BLAKE3] 1.2.0 with the following license: -------------------------------------------------------------------------------- +---- This work is released into the public domain with CC0 1.0. Alternatively, it is licensed under the Apache License 2.0. ------------------------------------------------------------------------------- -------------------------------------------------------------------------------- Creative Commons Legal Code @@ -211,7 +204,6 @@ this CC0 or use of the Work. ------------------------------------------------------------------------------- -------------------------------------------------------------------------------- Apache License Version 2.0, January 2004 @@ -414,16 +406,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- +---- -src/third_party/doctest.h -~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/doctest.h This is the single header version of https://github.com/onqtam/doctest[doctest] 2.4.6 with the following license: -------------------------------------------------------------------------------- +---- The MIT License (MIT) Copyright (c) 2016-2021 Viktor Kirilov @@ -445,15 +436,14 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- -src/third_party/fmt/*.h and src/third_party/format.cpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/fmt/*.h and src/third_party/format.cpp -This is a subset of https://fmt.dev[fmt] 7.1.3 with the following license: +This is a subset of https://fmt.dev[fmt] 8.0.1 with the following license: -------------------------------------------------------------------------------- +---- Formatting library for C++ Copyright (c) 2012 - present, Victor Zverovich @@ -482,16 +472,15 @@ of this Software are embedded into a machine-executable object form of such source code, you may redistribute such embedded portions in such object form without including the above copyright and permission notices. -------------------------------------------------------------------------------- +---- -src/third_party/getopt_long.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/getopt_long.* This implementation of `getopt_long()` was copied from https://www.postgresql.org[PostgreSQL] and has the following license text: -------------------------------------------------------------------------------- +---- Portions Copyright (c) 1987, 1993, 1994 The Regents of the University of California. All rights reserved. @@ -521,16 +510,46 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- +---- + + +=== src/third_party/httplib.* + +cpp-httplib - A C++11 cross-platform HTTP/HTTPS library. Copied from cpp-httplib +v0.9.7 downloaded from https://github.com/yhirose/cpp-httplib[cpp-httplib]. The +library has the following license: + +---- +The MIT License (MIT) + +Copyright (c) 2021 yhirose + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +---- -src/third_party/minitrace.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/minitrace.* A library for producing JSON traces suitable for Chrome's built-in trace viewer (chrome://tracing). Downloaded from . -------------------------------------------------------------------------------- +---- The MIT License (MIT) Copyright (c) 2014 Henrik Rydgård @@ -552,18 +571,17 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- -src/third_party/nonstd/optional.hpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/nonstd/expected.hpp This is the single header version of -https://github.com/martinmoene/optional-lite[optional-lite] 3.4.0 with the +https://github.com/martinmoene/expected-lite[expected-lite] 0.5.0 with the following license: -------------------------------------------------------------------------------- -Copyright (c) 2014-2018 Martin Moene +---- +Copyright (c) 2016-2018 Martin Moene Boost Software License - Version 1.0 - August 17th, 2003 @@ -588,17 +606,51 @@ FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- + + +=== src/third_party/nonstd/optional.hpp + +This is the single header version of +https://github.com/martinmoene/optional-lite[optional-lite] 3.5.0 with the +following license: + +---- +Copyright (c) 2014-2021 Martin Moene + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. -src/third_party/nonstd/string_view.hpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +---- + + +=== src/third_party/nonstd/string_view.hpp This alternative implementation of `std::string_view` was downloaded from and has the following license text: -------------------------------------------------------------------------------- +---- Copyright 2017-2020 by Martin Moene Boost Software License - Version 1.0 - August 17th, 2003 @@ -624,11 +676,40 @@ FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- + + +=== src/third_party/url.* + +CxxUrl - A simple C++ URL class. Copied from CxxUrl v0.3 downloaded from +. It has the following license text: + +---- +The MIT License (MIT) +Copyright (c) 2015 Christophe Meessen -src/third_party/win32/getopt.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +---- + + +=== src/third_party/win32/getopt.* This implementation of `getopt_long()` for Win32 was taken from https://www.codeproject.com/Articles/157001/Full-getopt-Port-for-Unicode-and-Multibyte-Microso @@ -638,14 +719,13 @@ https://www.gnu.org/licenses/lgpl-3.0.html. -src/third_party/win32/mktemp.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/win32/mktemp.* This implementation of `mkstemp()` for Win32 was adapted from -and has the folowing license text: +and has the following license text: -------------------------------------------------------------------------------- +---- Copyright (c) 1996-1998, 2008 Theo de Raadt Copyright (c) 1997, 2008-2009 Todd C. Miller @@ -660,16 +740,16 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -------------------------------------------------------------------------------- +---- -src/third_party/win32/winerror_to_errno.h -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +=== src/third_party/win32/winerror_to_errno.h The implementation of `winerror_to_errno()` was adapted from -and has the folowing license text: +and has the following license text: -------------------------------------------------------------------------------- +---- PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 1. This LICENSE AGREEMENT is between the Python Software Foundation @@ -717,15 +797,15 @@ 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. -------------------------------------------------------------------------------- +---- -src/third_party/xxh* -~~~~~~~~~~~~~~~~~~~~ + +=== src/third_party/xxh* xxHash - Extremely Fast Hash algorithm. Copied from xxHash v0.8.0 downloaded from . -------------------------------------------------------------------------------- +---- Copyright (c) 2012-2020 Yann Collet All rights reserved. @@ -753,4 +833,4 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- +---- diff -Nru ccache-4.2.1/.mailmap ccache-4.5.1/.mailmap --- ccache-4.2.1/.mailmap 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/.mailmap 2021-11-17 19:31:58.000000000 +0000 @@ -4,6 +4,7 @@ Andrew Boie Arne Hasselbring Bernhard Bauer +Bin Li Chiaki Ishikawa Clemens Rabe Clemens Rabe @@ -24,6 +25,7 @@ Ramiro Polla Ramiro Polla Ryan Brown +Ryan Burns <52847440+r-burns@users.noreply.github.com> Thomas Otto <39962140+totph@users.noreply.github.com> Thomas Röfer Tor Arne Vestbø diff -Nru ccache-4.2.1/misc/ccache.magic ccache-4.5.1/misc/ccache.magic --- ccache-4.2.1/misc/ccache.magic 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/misc/ccache.magic 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,4 @@ +0 beshort 0xCCAC ccache entry +>2 byte x \b, format version %d +>3 byte 0 \b, result +>3 byte 1 \b, manifest diff -Nru ccache-4.2.1/misc/clang-format ccache-4.5.1/misc/clang-format --- ccache-4.2.1/misc/clang-format 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/misc/clang-format 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,65 @@ +#!/bin/sh +# +# This script executes clang-format in the following order: +# +# 1. If environment variable CLANG_FORMAT is set, execute $CLANG_FORMAT. +# 2. Otherwise, if /misc/.clang-format-exe exists, execute that +# program. +# 3. Otherwise, download a statically linked clang-format executable, verify its +# integrity, place it in /misc/.clang-format-exe and execute +# it. + +set -eu + +if [ -n "${CLANG_FORMAT:-}" ]; then + exec "$CLANG_FORMAT" "$@" +fi + +top_dir="$(dirname "$0")" +clang_format_exe="$top_dir/.clang-format-exe" +clang_format_version=10 +url_prefix="https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-${clang_format_version}_" + +if [ ! -x "$clang_format_exe" ]; then + case "$(uname -s | tr '[:upper:]' '[:lower:]')" in + *mingw*|*cygwin*|*msys*) + url_suffix=windows-amd64.exe + checksum=0b21dfb9041437eebcc44032096e4dfba9ee7b668ee6867ada71ea3541f7b09657ea592a5b1cf659bc9f8072bdc477dfc9da07b3edacb7834b70e68d7a3fc887 + ;; + *darwin*) + url_suffix=macosx-amd64 + checksum=8458753e13d3cbb7949a302beab812bed6d9dd9001c6e9618e5ba2e438233633ae04704a4e150aa2abfbaf103f1df4bc4a77b306012f44b37e543964bd527951 + ;; + *linux*) + url_suffix=linux-amd64 + checksum=3c4aaa90ad83313a1d7b350b30f9ad62578be73241f00f7d6e92838289e0b734fab80dee9dfcbd1546135bdb4e3601cfb2cf6b47360fba0bfc961e5a7cab2015 + ;; + *) + echo "Error: Please set CLANG_FORMAT to clang-format version $clang_format_version" >&2 + exit 1 + ;; + esac + + url="$url_prefix$url_suffix" + + if command -v wget >/dev/null; then + wget -qO "$clang_format_exe.tmp" "$url" + elif command -v curl >/dev/null; then + curl -so "$clang_format_exe.tmp" -L --retry 20 "$url" + else + echo "Error: Neither wget nor curl found" >&2 + exit 1 + fi + + if ! command -v sha512sum >/dev/null; then + echo "Warning: sha512sum not found, not verifying clang-format integrity" >&2 + elif ! echo "$checksum $clang_format_exe.tmp" | sha512sum --status -c; then + echo "Error: Bad checksum of downloaded clang-format" >&2 + exit 1 + fi + + chmod +x "$clang_format_exe.tmp" + mv "$clang_format_exe.tmp" "$clang_format_exe" +fi + +exec "$clang_format_exe" "$@" diff -Nru ccache-4.2.1/misc/codespell-allowlist.txt ccache-4.5.1/misc/codespell-allowlist.txt --- ccache-4.2.1/misc/codespell-allowlist.txt 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/misc/codespell-allowlist.txt 2021-11-17 19:31:58.000000000 +0000 @@ -1,6 +1,7 @@ copyable creat files' +hda pase seve stoll diff -Nru ccache-4.2.1/misc/format-files ccache-4.5.1/misc/format-files --- ccache-4.2.1/misc/format-files 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/misc/format-files 2021-11-17 19:31:58.000000000 +0000 @@ -27,7 +27,7 @@ exec sh "$0" $check $(git ls-files '*.[ch]' '*.[ch]pp' ':!:src/third_party') fi -clang_format=${CLANG_FORMAT:-clang-format} +clang_format="$(dirname "$0")/clang-format" [ -t 1 ] && cf_color="--color=always" || cf_color="" if [ -n "${VERBOSE:-}" ]; then diff -Nru ccache-4.2.1/misc/performance ccache-4.5.1/misc/performance --- ccache-4.2.1/misc/performance 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/misc/performance 2021-11-17 19:31:58.000000000 +0000 @@ -263,9 +263,7 @@ op.add_option( "--no-compression", help="disable compression", action="store_true" ) - op.add_option( - "--compression-level", help="set compression level", type=int - ) + op.add_option("--compression-level", help="set compression level", type=int) op.add_option( "-d", "--directory", diff -Nru ccache-4.2.1/misc/test-all-systems ccache-4.5.1/misc/test-all-systems --- ccache-4.2.1/misc/test-all-systems 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/misc/test-all-systems 2021-11-17 19:31:58.000000000 +0000 @@ -20,17 +20,11 @@ # NAME CC CXX TEST_CC CMAKE_PARAMS -build debian-9 gcc g++ gcc -build debian-9 clang clang++ clang - build debian-10 gcc g++ gcc build debian-10 clang clang++ clang -build ubuntu-14.04 gcc g++ gcc -DZSTD_FROM_INTERNET=ON -build ubuntu-14.04 gcc g++ clang -DZSTD_FROM_INTERNET=ON - -build ubuntu-16.04 gcc g++ gcc -build ubuntu-16.04 clang clang++ clang +build debian-11 gcc g++ gcc +build debian-11 clang clang++ clang build ubuntu-18.04 gcc g++ gcc build ubuntu-18.04 clang clang++ clang @@ -38,8 +32,8 @@ build ubuntu-20.04 gcc g++ gcc build ubuntu-20.04 clang clang++ clang -build centos-7 gcc g++ gcc -DWARNINGS_AS_ERRORS=false -build centos-7 gcc g++ clang -DWARNINGS_AS_ERRORS=false +build centos-7 gcc g++ gcc -DWARNINGS_AS_ERRORS=false -DREDIS_STORAGE_BACKEND=OFF +build centos-7 gcc g++ clang -DWARNINGS_AS_ERRORS=false -DREDIS_STORAGE_BACKEND=OFF build centos-8 gcc g++ gcc build centos-8 clang clang++ clang @@ -47,8 +41,8 @@ build fedora-32 gcc g++ gcc build fedora-32 clang clang++ clang -build alpine-3.4 gcc g++ gcc -DZSTD_FROM_INTERNET=ON -build alpine-3.4 gcc g++ clang -DZSTD_FROM_INTERNET=ON +build alpine-3.8 gcc g++ gcc +build alpine-3.8 gcc g++ clang -build alpine-3.12 gcc g++ gcc -build alpine-3.12 clang clang++ clang +build alpine-3.14 gcc g++ gcc +build alpine-3.14 clang clang++ clang diff -Nru ccache-4.2.1/misc/upload-redis ccache-4.5.1/misc/upload-redis --- ccache-4.2.1/misc/upload-redis 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/misc/upload-redis 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +# This script uploads the contents of the cache from primary storage to a Redis +# secondary storage. + +import redis +import os + +config = os.getenv("REDIS_CONF", "localhost") +if ":" in config: + host, port = config.rsplit(":", 1) + sock = None +elif config.startswith("/"): + host, port, sock = None, None, config +else: + host, port, sock = config, 6379, None +username = os.getenv("REDIS_USERNAME") +password = os.getenv("REDIS_PASSWORD") +context = redis.Redis( + host=host, port=port, unix_socket_path=sock, password=password +) + +CCACHE_MANIFEST = b"cCmF" +CCACHE_RESULT = b"cCrS" + +ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.cache/ccache")) +filelist = [] +for dirpath, dirnames, filenames in os.walk(ccache): + # sort by modification time, most recently used last + for filename in filenames: + if filename.endswith(".lock"): + continue + stat = os.stat(os.path.join(dirpath, filename)) + filelist.append((stat.st_mtime, dirpath, filename)) +filelist.sort() +files = result = manifest = objects = 0 +for mtime, dirpath, filename in filelist: + dirname = dirpath.replace(ccache + os.path.sep, "") + if dirname == "tmp": + continue + elif filename == "CACHEDIR.TAG" or filename == "stats": + # ignore these + files = files + 1 + else: + (base, ext) = filename[:-1], filename[-1:] + if ext == "R" or ext == "M": + if ext == "R": + result = result + 1 + if ext == "M": + manifest = manifest + 1 + key = "ccache:" + "".join(list(os.path.split(dirname)) + [base]) + val = open(os.path.join(dirpath, filename), "rb").read() or None + if val: + print("%s: %s %d" % (key, ext, len(val))) + magic = val[0:4] + if ext == "M": + assert magic == CCACHE_MANIFEST + if ext == "R": + assert magic == CCACHE_RESULT + context.set(key, val) + objects = objects + 1 + files = files + 1 +print( + "%d files, %d result (%d manifest) = %d objects" + % (files, result, manifest, objects) +) diff -Nru ccache-4.2.1/README.md ccache-4.5.1/README.md --- ccache-4.2.1/README.md 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/README.md 2021-11-17 19:31:58.000000000 +0000 @@ -4,6 +4,7 @@ [![Build Status](https://github.com/ccache/ccache/workflows/Build/badge.svg)](https://github.com/ccache/ccache/actions?query=workflow%3A%22Build%22) [![Code Quality: Cpp](https://img.shields.io/lgtm/grade/cpp/g/ccache/ccache.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/ccache/ccache/context:cpp) [![Total Alerts](https://img.shields.io/lgtm/alerts/g/ccache/ccache.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/ccache/ccache/alerts) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5057/badge)](https://bestpractices.coreinfrastructure.org/projects/5057) [![Gitter](https://img.shields.io/gitter/room/ccache/ccache.svg)](https://gitter.im/ccache/ccache) Ccache (or “ccache”) is a compiler cache. It [speeds up diff -Nru ccache-4.2.1/src/argprocessing.cpp ccache-4.5.1/src/argprocessing.cpp --- ccache-4.2.1/src/argprocessing.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/argprocessing.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -19,15 +19,22 @@ #include "argprocessing.hpp" #include "Context.hpp" -#include "FormatNonstdStringView.hpp" #include "Logging.hpp" #include "assertions.hpp" #include "compopt.hpp" #include "fmtmacros.hpp" #include "language.hpp" +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #include +using core::Statistic; using nonstd::nullopt; using nonstd::optional; using nonstd::string_view; @@ -46,6 +53,7 @@ ColorDiagnostics color_diagnostics = ColorDiagnostics::automatic; bool found_directives_only = false; bool found_rewrite_includes = false; + nonstd::optional found_xarch_arch; std::string explicit_language; // As specified with -x. std::string input_charset_option; // -finput-charset=... @@ -93,13 +101,14 @@ color_output_possible() { const char* term_env = getenv("TERM"); - return isatty(STDERR_FILENO) && term_env && strcasecmp(term_env, "DUMB") != 0; + return isatty(STDERR_FILENO) && term_env + && Util::to_lowercase(term_env) != "dumb"; } bool -detect_pch(Context& ctx, - const std::string& option, +detect_pch(const std::string& option, const std::string& arg, + std::string& included_pch_file, bool is_cc1_option, bool* found_pch) { @@ -126,20 +135,22 @@ } if (!pch_file.empty()) { - if (!ctx.included_pch_file.empty()) { + if (!included_pch_file.empty()) { LOG("Multiple precompiled headers used: {} and {}", - ctx.included_pch_file, + included_pch_file, pch_file); return false; } - ctx.included_pch_file = pch_file; + included_pch_file = pch_file; *found_pch = true; } return true; } bool -process_profiling_option(Context& ctx, const std::string& arg) +process_profiling_option(const Context& ctx, + ArgsInfo& args_info, + const std::string& arg) { static const std::vector known_simple_options = { "-fprofile-correction", @@ -156,31 +167,31 @@ std::string new_profile_path; bool new_profile_use = false; - if (Util::starts_with(arg, "-fprofile-dir=")) { + if (util::starts_with(arg, "-fprofile-dir=")) { new_profile_path = arg.substr(arg.find('=') + 1); } else if (arg == "-fprofile-generate" || arg == "-fprofile-instr-generate") { - ctx.args_info.profile_generate = true; + args_info.profile_generate = true; if (ctx.config.compiler_type() == CompilerType::clang) { new_profile_path = "."; } else { // GCC uses $PWD/$(basename $obj). new_profile_path = ctx.apparent_cwd; } - } else if (Util::starts_with(arg, "-fprofile-generate=") - || Util::starts_with(arg, "-fprofile-instr-generate=")) { - ctx.args_info.profile_generate = true; + } else if (util::starts_with(arg, "-fprofile-generate=") + || util::starts_with(arg, "-fprofile-instr-generate=")) { + args_info.profile_generate = true; new_profile_path = arg.substr(arg.find('=') + 1); } else if (arg == "-fprofile-use" || arg == "-fprofile-instr-use" || arg == "-fprofile-sample-use" || arg == "-fbranch-probabilities" || arg == "-fauto-profile") { new_profile_use = true; - if (ctx.args_info.profile_path.empty()) { + if (args_info.profile_path.empty()) { new_profile_path = "."; } - } else if (Util::starts_with(arg, "-fprofile-use=") - || Util::starts_with(arg, "-fprofile-instr-use=") - || Util::starts_with(arg, "-fprofile-sample-use=") - || Util::starts_with(arg, "-fauto-profile=")) { + } else if (util::starts_with(arg, "-fprofile-use=") + || util::starts_with(arg, "-fprofile-instr-use=") + || util::starts_with(arg, "-fprofile-sample-use=") + || util::starts_with(arg, "-fauto-profile=")) { new_profile_use = true; new_profile_path = arg.substr(arg.find('=') + 1); } else { @@ -189,19 +200,19 @@ } if (new_profile_use) { - if (ctx.args_info.profile_use) { + if (args_info.profile_use) { LOG_RAW("Multiple profiling options not supported"); return false; } - ctx.args_info.profile_use = true; + args_info.profile_use = true; } if (!new_profile_path.empty()) { - ctx.args_info.profile_path = new_profile_path; - LOG("Set profile directory to {}", ctx.args_info.profile_path); + args_info.profile_path = new_profile_path; + LOG("Set profile directory to {}", args_info.profile_path); } - if (ctx.args_info.profile_generate && ctx.args_info.profile_use) { + if (args_info.profile_generate && args_info.profile_use) { // Too hard to figure out what the compiler will do. LOG_RAW("Both generating and using profile info, giving up"); return false; @@ -211,14 +222,13 @@ } optional -process_arg(Context& ctx, +process_arg(const Context& ctx, + ArgsInfo& args_info, + Config& config, Args& args, size_t& args_index, ArgumentProcessingState& state) { - ArgsInfo& args_info = ctx.args_info; - Config& config = ctx.config; - size_t& i = args_index; // The user knows best: just swallow the next arg. @@ -232,13 +242,22 @@ return nullopt; } + // Ignore clang -ivfsoverlay to not detect multiple input files. + if (args[i] == "-ivfsoverlay" + && !(config.sloppiness().is_enabled(core::Sloppy::ivfsoverlay))) { + LOG_RAW( + "You have to specify \"ivfsoverlay\" sloppiness when using" + " -ivfsoverlay to get hits"); + return Statistic::unsupported_compiler_option; + } + // Special case for -E. if (args[i] == "-E") { return Statistic::called_for_preprocessing; } // Handle "@file" argument. - if (Util::starts_with(args[i], "@") || Util::starts_with(args[i], "-@")) { + if (util::starts_with(args[i], "@") || util::starts_with(args[i], "-@")) { const char* argpath = args[i].c_str() + 1; if (argpath[-1] == '-') { @@ -280,8 +299,8 @@ } // These are always too hard. - if (compopt_too_hard(args[i]) || Util::starts_with(args[i], "-fdump-") - || Util::starts_with(args[i], "-MJ")) { + if (compopt_too_hard(args[i]) || util::starts_with(args[i], "-fdump-") + || util::starts_with(args[i], "-MJ")) { LOG("Compiler option {} is unsupported", args[i]); return Statistic::unsupported_compiler_option; } @@ -293,9 +312,21 @@ } // -Xarch_* options are too hard. - if (Util::starts_with(args[i], "-Xarch_")) { - LOG("Unsupported compiler option: {}", args[i]); - return Statistic::unsupported_compiler_option; + if (util::starts_with(args[i], "-Xarch_")) { + if (i == args.size() - 1) { + LOG("Missing argument to {}", args[i]); + return Statistic::bad_compiler_arguments; + } + const auto arch = args[i].substr(7); + if (!state.found_xarch_arch) { + state.found_xarch_arch = arch; + } else if (*state.found_xarch_arch != arch) { + LOG_RAW("Multiple different -Xarch_* options not supported"); + return Statistic::unsupported_compiler_option; + } + state.common_args.push_back(args[i]); + state.common_args.push_back(args[i + 1]); + return nullopt; } // Handle -arch options. @@ -311,7 +342,7 @@ // Some arguments that clang passes directly to cc1 (related to precompiled // headers) need the usual ccache handling. In those cases, the -Xclang // prefix is skipped and the cc1 argument is handled instead. - if (args[i] == "-Xclang" && i < args.size() - 1 + if (args[i] == "-Xclang" && i + 1 < args.size() && (args[i + 1] == "-emit-pch" || args[i + 1] == "-emit-pth" || args[i + 1] == "-include-pch" || args[i + 1] == "-include-pth" || args[i + 1] == "-fno-pch-timestamp")) { @@ -358,7 +389,7 @@ LOG("Compiler option {} is unsupported without direct depend mode", args[i]); return Statistic::could_not_use_modules; - } else if (!(config.sloppiness() & SLOPPY_MODULES)) { + } else if (!(config.sloppiness().is_enabled(core::Sloppy::modules))) { LOG_RAW( "You have to specify \"modules\" sloppiness when using" " -fmodules to get hits"); @@ -386,7 +417,7 @@ return nullopt; } - if (Util::starts_with(args[i], "-x")) { + if (util::starts_with(args[i], "-x")) { if (args[i].length() >= 3 && !islower(args[i][2])) { // -xCODE (where CODE can be e.g. Host or CORE-AVX2, always starting with // an uppercase letter) is an ordinary Intel compiler option, not a @@ -429,15 +460,15 @@ } // Alternate form of -o with no space. Nvcc does not support this. - if (Util::starts_with(args[i], "-o") + if (util::starts_with(args[i], "-o") && config.compiler_type() != CompilerType::nvcc) { args_info.output_obj = Util::make_relative_path(ctx, string_view(args[i]).substr(2)); return nullopt; } - if (Util::starts_with(args[i], "-fdebug-prefix-map=") - || Util::starts_with(args[i], "-ffile-prefix-map=")) { + if (util::starts_with(args[i], "-fdebug-prefix-map=") + || util::starts_with(args[i], "-ffile-prefix-map=")) { std::string map = args[i].substr(args[i].find('=') + 1); args_info.debug_prefix_maps.push_back(map); state.common_args.push_back(args[i]); @@ -446,17 +477,17 @@ // Debugging is handled specially, so that we know if we can strip line // number info. - if (Util::starts_with(args[i], "-g")) { + if (util::starts_with(args[i], "-g")) { state.common_args.push_back(args[i]); - if (Util::starts_with(args[i], "-gdwarf")) { + if (util::starts_with(args[i], "-gdwarf")) { // Selection of DWARF format (-gdwarf or -gdwarf-) enables // debug info on level 2. args_info.generating_debuginfo = true; return nullopt; } - if (Util::starts_with(args[i], "-gz")) { + if (util::starts_with(args[i], "-gz")) { // -gz[=type] neither disables nor enables debug info. return nullopt; } @@ -487,7 +518,7 @@ return nullopt; } - if (Util::starts_with(args[i], "-MF")) { + if (util::starts_with(args[i], "-MF")) { state.dependency_filename_specified = true; std::string dep_file; @@ -515,8 +546,8 @@ return nullopt; } - if (Util::starts_with(args[i], "-MQ") || Util::starts_with(args[i], "-MT")) { - ctx.args_info.dependency_target_specified = true; + if (util::starts_with(args[i], "-MQ") || util::starts_with(args[i], "-MT")) { + args_info.dependency_target_specified = true; if (args[i].size() == 3) { // -MQ arg or -MT arg @@ -569,10 +600,10 @@ return nullopt; } - if (Util::starts_with(args[i], "-fprofile-") - || Util::starts_with(args[i], "-fauto-profile") + if (util::starts_with(args[i], "-fprofile-") + || util::starts_with(args[i], "-fauto-profile") || args[i] == "-fbranch-probabilities") { - if (!process_profiling_option(ctx, args[i])) { + if (!process_profiling_option(ctx, args_info, args[i])) { // The failure is logged by process_profiling_option. return Statistic::unsupported_compiler_option; } @@ -580,13 +611,13 @@ return nullopt; } - if (Util::starts_with(args[i], "-fsanitize-blacklist=")) { + if (util::starts_with(args[i], "-fsanitize-blacklist=")) { args_info.sanitize_blacklists.emplace_back(args[i].substr(21)); state.common_args.push_back(args[i]); return nullopt; } - if (Util::starts_with(args[i], "--sysroot=")) { + if (util::starts_with(args[i], "--sysroot=")) { auto path = string_view(args[i]).substr(10); auto relpath = Util::make_relative_path(ctx, path); state.common_args.push_back("--sysroot=" + relpath); @@ -618,15 +649,21 @@ return nullopt; } - if (Util::starts_with(args[i], "-Wp,")) { - if (args[i] == "-Wp,-P" || args[i].find(",-P,") != std::string::npos - || Util::ends_with(args[i], ",-P")) { - // -P removes preprocessor information in such a way that the object file - // from compiling the preprocessed file will not be equal to the object - // file produced when compiling without ccache. - LOG_RAW("Too hard option -Wp,-P detected"); + if (args[i] == "-P" || args[i] == "-Wp,-P") { + // Avoid passing -P to the preprocessor since it removes preprocessor + // information we need. + state.compiler_only_args.push_back(args[i]); + LOG("{} used; not compiling preprocessed code", args[i]); + config.set_run_second_cpp(true); + return nullopt; + } + + if (util::starts_with(args[i], "-Wp,")) { + if (args[i].find(",-P,") != std::string::npos + || util::ends_with(args[i], ",-P")) { + // -P together with other preprocessor options is just too hard. return Statistic::unsupported_compiler_option; - } else if (Util::starts_with(args[i], "-Wp,-MD,") + } else if (util::starts_with(args[i], "-Wp,-MD,") && args[i].find(',', 8) == std::string::npos) { args_info.generating_dependencies = true; state.dependency_filename_specified = true; @@ -634,7 +671,7 @@ Util::make_relative_path(ctx, string_view(args[i]).substr(8)); state.dep_args.push_back(args[i]); return nullopt; - } else if (Util::starts_with(args[i], "-Wp,-MMD,") + } else if (util::starts_with(args[i], "-Wp,-MMD,") && args[i].find(',', 9) == std::string::npos) { args_info.generating_dependencies = true; state.dependency_filename_specified = true; @@ -642,13 +679,13 @@ Util::make_relative_path(ctx, string_view(args[i]).substr(9)); state.dep_args.push_back(args[i]); return nullopt; - } else if (Util::starts_with(args[i], "-Wp,-D") + } else if (util::starts_with(args[i], "-Wp,-D") && args[i].find(',', 6) == std::string::npos) { // Treat it like -D. state.cpp_args.push_back(args[i].substr(4)); return nullopt; } else if (args[i] == "-Wp,-MP" - || (args[i].size() > 8 && Util::starts_with(args[i], "-Wp,-M") + || (args[i].size() > 8 && util::starts_with(args[i], "-Wp,-M") && args[i][7] == ',' && (args[i][6] == 'F' || args[i][6] == 'Q' || args[i][6] == 'T') @@ -674,7 +711,7 @@ } // Input charset needs to be handled specially. - if (Util::starts_with(args[i], "-finput-charset=")) { + if (util::starts_with(args[i], "-finput-charset=")) { state.input_charset_option = args[i]; return nullopt; } @@ -705,7 +742,7 @@ // In the "-Xclang -fcolor-diagnostics" form, -Xclang is skipped and the // -fcolor-diagnostics argument which is passed to cc1 is handled below. - if (args[i] == "-Xclang" && i < args.size() - 1 + if (args[i] == "-Xclang" && i + 1 < args.size() && args[i + 1] == "-fcolor-diagnostics") { state.compiler_only_args_no_hash.push_back(args[i]); ++i; @@ -753,7 +790,7 @@ return nullopt; } - if (config.sloppiness() & SLOPPY_CLANG_INDEX_STORE + if (config.sloppiness().is_enabled(core::Sloppy::clang_index_store) && args[i] == "-index-store-path") { // Xcode 9 or later calls Clang with this option. The given path includes a // UUID that might lead to cache misses, especially when cache is shared @@ -785,8 +822,11 @@ next = 2; } - if (!detect_pch( - ctx, args[i], args[i + next], next == 2, &state.found_pch)) { + if (!detect_pch(args[i], + args[i + next], + args_info.included_pch_file, + next == 2, + &state.found_pch)) { return Statistic::bad_compiler_arguments; } @@ -972,7 +1012,8 @@ optional argument_error; for (size_t i = 1; i < args.size(); i++) { - const auto error = process_arg(ctx, args, i, state); + const auto error = + process_arg(ctx, ctx.args_info, ctx.config, args, i, state); if (error && !argument_error) { argument_error = error; } @@ -1024,7 +1065,7 @@ if (state.found_pch || state.found_fpch_preprocess) { args_info.using_precompiled_header = true; - if (!(config.sloppiness() & SLOPPY_TIME_MACROS)) { + if (!(config.sloppiness().is_enabled(core::Sloppy::time_macros))) { LOG_RAW( "You have to specify \"time_macros\" sloppiness when using" " precompiled headers to get direct hits"); @@ -1060,7 +1101,7 @@ } if (args_info.output_is_precompiled_header - && !(config.sloppiness() & SLOPPY_PCH_DEFINES)) { + && !(config.sloppiness().is_enabled(core::Sloppy::pch_defines))) { LOG_RAW( "You have to specify \"pch_defines,time_macros\" sloppiness when" " creating precompiled headers"); @@ -1107,7 +1148,14 @@ } if (args_info.seen_split_dwarf) { - args_info.output_dwo = Util::change_extension(args_info.output_obj, ".dwo"); + if (args_info.output_obj == "/dev/null") { + // Outputting to /dev/null -> compiler won't write a .dwo, so just pretend + // we haven't seen the -gsplit-dwarf option. + args_info.seen_split_dwarf = false; + } else { + args_info.output_dwo = + Util::change_extension(args_info.output_obj, ".dwo"); + } } // Cope with -o /dev/null. @@ -1232,6 +1280,17 @@ compiler_args.push_back("-dc"); } + if (state.found_xarch_arch && !args_info.arch_args.empty()) { + if (args_info.arch_args.size() > 1) { + LOG_RAW( + "Multiple -arch options in combination with -Xarch_* not supported"); + return Statistic::unsupported_compiler_option; + } else if (args_info.arch_args[0] != *state.found_xarch_arch) { + LOG_RAW("-arch option not matching -Xarch_* option not supported"); + return Statistic::unsupported_compiler_option; + } + } + for (const auto& arch : args_info.arch_args) { compiler_args.push_back("-arch"); compiler_args.push_back(arch); diff -Nru ccache-4.2.1/src/argprocessing.hpp ccache-4.5.1/src/argprocessing.hpp --- ccache-4.2.1/src/argprocessing.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/argprocessing.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,7 +19,8 @@ #pragma once #include "Args.hpp" -#include "Statistic.hpp" + +#include #include "third_party/nonstd/optional.hpp" @@ -27,14 +28,14 @@ struct ProcessArgsResult { - ProcessArgsResult(Statistic error_); + ProcessArgsResult(core::Statistic error_); ProcessArgsResult(const Args& preprocessor_args_, const Args& extra_args_to_hash_, const Args& compiler_args_); // nullopt on success, otherwise the statistics counter that should be // incremented. - nonstd::optional error; + nonstd::optional error; // Arguments (except -E) to send to the preprocessor. Args preprocessor_args; @@ -46,7 +47,8 @@ Args compiler_args; }; -inline ProcessArgsResult::ProcessArgsResult(Statistic error_) : error(error_) +inline ProcessArgsResult::ProcessArgsResult(core::Statistic error_) + : error(error_) { } diff -Nru ccache-4.2.1/src/Args.cpp ccache-4.5.1/src/Args.cpp --- ccache-4.2.1/src/Args.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Args.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,6 +20,9 @@ #include "Util.hpp" +#include +#include + using nonstd::nullopt; using nonstd::optional; using nonstd::string_view; @@ -52,7 +55,7 @@ std::string argtext; try { argtext = Util::read_file(filename); - } catch (Error&) { + } catch (core::Error&) { return nullopt; } @@ -169,8 +172,8 @@ { m_args.erase(std::remove_if(m_args.begin(), m_args.end(), - [&prefix](const std::string& s) { - return Util::starts_with(s, prefix); + [&prefix](const auto& s) { + return util::starts_with(s, prefix); }), m_args.end()); } diff -Nru ccache-4.2.1/src/Args.hpp ccache-4.5.1/src/Args.hpp --- ccache-4.2.1/src/Args.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Args.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" #include "Util.hpp" @@ -115,18 +113,14 @@ return m_args.size(); } -// clang-format off inline const std::string& Args::operator[](size_t i) const -// clang-format on { return m_args[i]; } -// clang-format off inline std::string& Args::operator[](size_t i) -// clang-format on { return m_args[i]; } diff -Nru ccache-4.2.1/src/ArgsInfo.hpp ccache-4.5.1/src/ArgsInfo.hpp --- ccache-4.2.1/src/ArgsInfo.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ArgsInfo.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "Args.hpp" #include @@ -51,6 +49,9 @@ // Split dwarf information (GCC 4.8 and up). Contains pathname if not empty. std::string output_dwo; + // The .gch/.pch/.pth file used for compilation. + std::string included_pch_file; + // Language to use for the compilation target (see language.c). std::string actual_language; diff -Nru ccache-4.2.1/src/assertions.hpp ccache-4.5.1/src/assertions.hpp --- ccache-4.2.1/src/assertions.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/assertions.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,7 +18,7 @@ #pragma once -#include "system.hpp" +#include #ifdef _MSC_VER # define CCACHE_FUNCTION __func__ diff -Nru ccache-4.2.1/src/AtomicFile.cpp ccache-4.5.1/src/AtomicFile.cpp --- ccache-4.2.1/src/AtomicFile.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/AtomicFile.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -21,7 +21,8 @@ #include "TemporaryFile.hpp" #include "Util.hpp" #include "assertions.hpp" -#include "exceptions.hpp" + +#include AtomicFile::AtomicFile(const std::string& path, Mode mode) : m_path(path) { @@ -43,7 +44,8 @@ AtomicFile::write(const std::string& data) { if (fwrite(data.data(), data.size(), 1, m_stream) != 1) { - throw Error("failed to write data to {}: {}", m_path, strerror(errno)); + throw core::Error( + "failed to write data to {}: {}", m_path, strerror(errno)); } } @@ -51,7 +53,8 @@ AtomicFile::write(const std::vector& data) { if (fwrite(data.data(), data.size(), 1, m_stream) != 1) { - throw Error("failed to write data to {}: {}", m_path, strerror(errno)); + throw core::Error( + "failed to write data to {}: {}", m_path, strerror(errno)); } } @@ -63,7 +66,8 @@ m_stream = nullptr; if (result == EOF) { Util::unlink_tmp(m_tmp_path); - throw Error("failed to write data to {}: {}", m_path, strerror(errno)); + throw core::Error( + "failed to write data to {}: {}", m_path, strerror(errno)); } Util::rename(m_tmp_path, m_path); } diff -Nru ccache-4.2.1/src/AtomicFile.hpp ccache-4.5.1/src/AtomicFile.hpp --- ccache-4.2.1/src/AtomicFile.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/AtomicFile.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,8 @@ #pragma once -#include "system.hpp" - +#include +#include #include #include diff -Nru ccache-4.2.1/src/CacheEntryReader.cpp ccache-4.5.1/src/CacheEntryReader.cpp --- ccache-4.2.1/src/CacheEntryReader.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/CacheEntryReader.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,94 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheEntryReader.hpp" - -#include "Compressor.hpp" -#include "exceptions.hpp" -#include "fmtmacros.hpp" - -#include "third_party/fmt/core.h" - -CacheEntryReader::CacheEntryReader(FILE* stream, - const uint8_t* expected_magic, - uint8_t expected_version) -{ - uint8_t header_bytes[15]; - if (fread(header_bytes, sizeof(header_bytes), 1, stream) != 1) { - throw Error("Error reading header"); - } - - memcpy(m_magic, header_bytes, sizeof(m_magic)); - m_version = header_bytes[4]; - m_compression_type = Compression::type_from_int(header_bytes[5]); - m_compression_level = header_bytes[6]; - Util::big_endian_to_int(header_bytes + 7, m_content_size); - - if (memcmp(m_magic, expected_magic, sizeof(m_magic)) != 0) { - throw Error("Bad magic value 0x{:02x}{:02x}{:02x}{:02x}", - m_magic[0], - m_magic[1], - m_magic[2], - m_magic[3]); - } - if (m_version != expected_version) { - throw Error( - "Unknown version (actual {}, expected {})", m_version, expected_version); - } - - m_checksum.update(header_bytes, sizeof(header_bytes)); - m_decompressor = Decompressor::create_from_type(m_compression_type, stream); -} - -void -CacheEntryReader::dump_header(FILE* dump_stream) -{ - PRINT(dump_stream, "Magic: {:.4}\n", m_magic); - PRINT(dump_stream, "Version: {}\n", m_version); - PRINT(dump_stream, - "Compression type: {}\n", - Compression::type_to_string(m_compression_type)); - PRINT(dump_stream, "Compression level: {}\n", m_compression_level); - PRINT(dump_stream, "Content size: {}\n", m_content_size); -} - -void -CacheEntryReader::read(void* data, size_t count) -{ - m_decompressor->read(data, count); - m_checksum.update(data, count); -} - -void -CacheEntryReader::finalize() -{ - uint64_t actual_digest = m_checksum.digest(); - - uint8_t buffer[8]; - read(buffer, sizeof(buffer)); - uint64_t expected_digest; - Util::big_endian_to_int(buffer, expected_digest); - - if (actual_digest != expected_digest) { - throw Error("Incorrect checksum (actual 0x{:016x}, expected 0x{:016x})", - actual_digest, - expected_digest); - } - - m_decompressor->finalize(); -} diff -Nru ccache-4.2.1/src/CacheEntryReader.hpp ccache-4.5.1/src/CacheEntryReader.hpp --- ccache-4.2.1/src/CacheEntryReader.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/CacheEntryReader.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,146 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Checksum.hpp" -#include "Decompressor.hpp" -#include "Util.hpp" - -#include - -// This class knows how to read a cache entry with a common header and a -// payload part that is different depending on the cache entry type (result or -// manifest). -class CacheEntryReader -{ -public: - // Constructor. - // - // Parameters: - // - stream: Stream to read header and payload from. - // - expected_magic: Expected file format magic (first four bytes of the - // file). - // - expected_version: Expected file format version. - CacheEntryReader(FILE* stream, - const uint8_t* expected_magic, - uint8_t expected_version); - - // Dump header information in text format. - // - // Parameters: - // - dump_stream: Stream to write to. - void dump_header(FILE* dump_stream); - - // Read data into a buffer from the payload. - // - // Parameters: - // - data: Buffer to write data to. - // - count: How many bytes to write. - // - // Throws Error on failure. - void read(void* data, size_t count); - - // Read an unsigned integer from the payload. - // - // Parameters: - // - value: Variable to write to. - // - // Throws Error on failure. - template void read(T& value); - - // Close for reading. - // - // This method potentially verifies the end state after reading the cache - // entry and throws Error if any integrity issues are found. - void finalize(); - - // Get size of the payload, - uint64_t payload_size() const; - - // Get content magic. - const uint8_t* magic() const; - - // Get content version. - uint8_t version() const; - - // Get compression type. - Compression::Type compression_type() const; - - // Get compression level. - int8_t compression_level() const; - - // Get size of the content (header + payload + checksum). - uint64_t content_size() const; - -private: - std::unique_ptr m_decompressor; - Checksum m_checksum; - uint8_t m_magic[4]; - uint8_t m_version; - Compression::Type m_compression_type; - int8_t m_compression_level; - uint64_t m_content_size; -}; - -template -inline void -CacheEntryReader::read(T& value) -{ - uint8_t buffer[sizeof(T)]; - read(buffer, sizeof(T)); - Util::big_endian_to_int(buffer, value); -} - -inline const uint8_t* -CacheEntryReader::magic() const -{ - return m_magic; -} - -inline uint8_t -CacheEntryReader::version() const -{ - return m_version; -} - -inline Compression::Type -CacheEntryReader::compression_type() const -{ - return m_compression_type; -} - -inline int8_t -CacheEntryReader::compression_level() const -{ - return m_compression_level; -} - -inline uint64_t -CacheEntryReader::payload_size() const -{ - return m_content_size - 15 - 8; -} - -inline uint64_t -CacheEntryReader::content_size() const -{ - return m_content_size; -} diff -Nru ccache-4.2.1/src/CacheEntryWriter.cpp ccache-4.5.1/src/CacheEntryWriter.cpp --- ccache-4.2.1/src/CacheEntryWriter.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/CacheEntryWriter.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,60 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheEntryWriter.hpp" - -CacheEntryWriter::CacheEntryWriter(FILE* stream, - const uint8_t* magic, - uint8_t version, - Compression::Type compression_type, - int8_t compression_level, - uint64_t payload_size) - // clang-format off - : m_compressor( - Compressor::create_from_type(compression_type, stream, compression_level) - ) -{ - // clang-format on - uint8_t header_bytes[15]; - memcpy(header_bytes, magic, 4); - header_bytes[4] = version; - header_bytes[5] = static_cast(compression_type); - header_bytes[6] = m_compressor->actual_compression_level(); - uint64_t content_size = 15 + payload_size + 8; - Util::int_to_big_endian(content_size, header_bytes + 7); - if (fwrite(header_bytes, sizeof(header_bytes), 1, stream) != 1) { - throw Error("Failed to write cache entry header"); - } - m_checksum.update(header_bytes, sizeof(header_bytes)); -} - -void -CacheEntryWriter::write(const void* data, size_t count) -{ - m_compressor->write(data, count); - m_checksum.update(data, count); -} - -void -CacheEntryWriter::finalize() -{ - uint8_t buffer[8]; - Util::int_to_big_endian(m_checksum.digest(), buffer); - m_compressor->write(buffer, sizeof(buffer)); - m_compressor->finalize(); -} diff -Nru ccache-4.2.1/src/CacheEntryWriter.hpp ccache-4.5.1/src/CacheEntryWriter.hpp --- ccache-4.2.1/src/CacheEntryWriter.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/CacheEntryWriter.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,86 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Checksum.hpp" -#include "Compressor.hpp" -#include "Util.hpp" - -#include - -// This class knows how to write a cache entry with a common header and a -// payload part that is different depending on the cache entry type (result or -// manifest). -class CacheEntryWriter -{ -public: - // Constructor. - // - // Parameters: - // - stream: Stream to write header + payload to. - // - magic: File format magic (first four bytes of the file). - // - version: File format version. - // - compression_type: Compression type to use. - // - compression_level: Compression level to use. - // - payload_size: Payload size. - CacheEntryWriter(FILE* stream, - const uint8_t* magic, - uint8_t version, - Compression::Type compression_type, - int8_t compression_level, - uint64_t payload_size); - - // Write data to the payload from a buffer. - // - // Parameters: - // - data: Data to write. - // - count: Size of data to write. - // - // Throws Error on failure. - void write(const void* data, size_t count); - - // Write an unsigned integer to the payload. - // - // Parameters: - // - value: Value to write. - // - // Throws Error on failure. - template void write(T value); - - // Close for writing. - // - // This method potentially verifies the end state after writing the cache - // entry and throws Error if any integrity issues are found. - void finalize(); - -private: - std::unique_ptr m_compressor; - Checksum m_checksum; -}; - -template -inline void -CacheEntryWriter::write(T value) -{ - uint8_t buffer[sizeof(T)]; - Util::int_to_big_endian(value, buffer); - write(buffer, sizeof(T)); -} diff -Nru ccache-4.2.1/src/CacheFile.cpp ccache-4.5.1/src/CacheFile.cpp --- ccache-4.2.1/src/CacheFile.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/CacheFile.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,45 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheFile.hpp" - -#include "Manifest.hpp" -#include "Result.hpp" -#include "Util.hpp" - -const Stat& -CacheFile::lstat() const -{ - if (!m_stat) { - m_stat = Stat::lstat(m_path); - } - - return *m_stat; -} - -CacheFile::Type -CacheFile::type() const -{ - if (Util::ends_with(m_path, Manifest::k_file_suffix)) { - return Type::manifest; - } else if (Util::ends_with(m_path, Result::k_file_suffix)) { - return Type::result; - } else { - return Type::unknown; - } -} diff -Nru ccache-4.2.1/src/CacheFile.hpp ccache-4.5.1/src/CacheFile.hpp --- ccache-4.2.1/src/CacheFile.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/CacheFile.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,53 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Stat.hpp" - -#include "third_party/nonstd/optional.hpp" - -#include - -class CacheFile -{ -public: - enum class Type { result, manifest, unknown }; - - explicit CacheFile(const std::string& path); - - const Stat& lstat() const; - const std::string& path() const; - Type type() const; - -private: - std::string m_path; - mutable nonstd::optional m_stat; -}; - -inline CacheFile::CacheFile(const std::string& path) : m_path(path) -{ -} - -inline const std::string& -CacheFile::path() const -{ - return m_path; -} diff -Nru ccache-4.2.1/src/ccache.cpp ccache-4.5.1/src/ccache.cpp --- ccache-4.2.1/src/ccache.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ccache.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -21,8 +21,6 @@ #include "Args.hpp" #include "ArgsInfo.hpp" -#include "Checksum.hpp" -#include "Compression.hpp" #include "Context.hpp" #include "Depfile.hpp" #include "Fd.hpp" @@ -34,191 +32,110 @@ #include "Logging.hpp" #include "Manifest.hpp" #include "MiniTrace.hpp" -#include "ProgressBar.hpp" #include "Result.hpp" -#include "ResultDumper.hpp" -#include "ResultExtractor.hpp" #include "ResultRetriever.hpp" #include "SignalHandler.hpp" -#include "Statistics.hpp" -#include "StdMakeUnique.hpp" #include "TemporaryFile.hpp" #include "UmaskScope.hpp" #include "Util.hpp" +#include "Win32Util.hpp" #include "argprocessing.hpp" -#include "cleanup.hpp" #include "compopt.hpp" -#include "compress.hpp" -#include "exceptions.hpp" #include "execute.hpp" #include "fmtmacros.hpp" #include "hashutil.hpp" #include "language.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "third_party/fmt/core.h" #include "third_party/nonstd/optional.hpp" #include "third_party/nonstd/string_view.hpp" -#ifdef HAVE_GETOPT_LONG -# include -#elif defined(_WIN32) -# include "third_party/win32/getopt.h" -#else -extern "C" { -# include "third_party/getopt_long.h" -} -#endif +#include -#ifdef _WIN32 -# include "Win32Util.hpp" +#ifdef HAVE_UNISTD_H +# include #endif #include #include #include +#include #ifndef MYNAME # define MYNAME "ccache" #endif const char CCACHE_NAME[] = MYNAME; +using core::Statistic; using nonstd::nullopt; using nonstd::optional; using nonstd::string_view; -constexpr const char VERSION_TEXT[] = - R"({} version {} - -Copyright (C) 2002-2007 Andrew Tridgell -Copyright (C) 2009-2021 Joel Rosdahl and other contributors - -See for a complete list of contributors. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 3 of the License, or (at your option) any later -version. -)"; - -constexpr const char USAGE_TEXT[] = - R"(Usage: - {} [options] - {} compiler [compiler options] - compiler [compiler options] (via symbolic link) - -Common options: - -c, --cleanup delete old files and recalculate size counters - (normally not needed as this is done - automatically) - -C, --clear clear the cache completely (except configuration) - --config-path PATH operate on configuration file PATH instead of the - default - -d, --directory PATH operate on cache directory PATH instead of the - default - --evict-older-than AGE remove files older than AGE (unsigned integer - with a d (days) or s (seconds) suffix) - -F, --max-files NUM set maximum number of files in cache to NUM (use - 0 for no limit) - -M, --max-size SIZE set maximum size of cache to SIZE (use 0 for no - limit); available suffixes: k, M, G, T (decimal) - and Ki, Mi, Gi, Ti (binary); default suffix: G - -X, --recompress LEVEL recompress the cache to level LEVEL (integer or - "uncompressed") using the Zstandard algorithm; - see "Cache compression" in the manual for details - -o, --set-config KEY=VAL set configuration item KEY to value VAL - -x, --show-compression show compression statistics - -p, --show-config show current configuration options in - human-readable format - -s, --show-stats show summary of configuration and statistics - counters in human-readable format - -z, --zero-stats zero statistics counters - - -h, --help print this help text - -V, --version print version and copyright information - -Options for scripting or debugging: - --checksum-file PATH print the checksum (64 bit XXH3) of the file at - PATH - --dump-manifest PATH dump manifest file at PATH in text format - --dump-result PATH dump result file at PATH in text format - --extract-result PATH extract data stored in result file at PATH to the - current working directory - -k, --get-config KEY print the value of configuration key KEY - --hash-file PATH print the hash (160 bit BLAKE3) of the file at - PATH - --print-stats print statistics counter IDs and corresponding - values in machine-parsable format - -See also the manual on . -)"; - -// How often (in seconds) to scan $CCACHE_DIR/tmp for left-over temporary -// files. -const int k_tempdir_cleanup_interval = 2 * 24 * 60 * 60; // 2 days - -// Maximum files per cache directory. This constant is somewhat arbitrarily -// chosen to be large enough to avoid unnecessary cache levels but small enough -// not to make esoteric file systems (with bad performance for large -// directories) too slow. It could be made configurable, but hopefully there -// will be no need to do that. -const uint64_t k_max_cache_files_per_directory = 2000; - -// Minimum number of cache levels ($CCACHE_DIR/1/2/stored_file). -const uint8_t k_min_cache_levels = 2; - -// Maximum number of cache levels ($CCACHE_DIR/1/2/3/stored_file). -// -// On a cache miss, (k_max_cache_levels - k_min_cache_levels + 1) cache lookups -// (i.e. stat system calls) will be performed for a cache entry. -// -// An assumption made here is that if a cache is so large that it holds more -// than 16^4 * k_max_cache_files_per_directory files then we can assume that the -// file system is sane enough to handle more than -// k_max_cache_files_per_directory. -const uint8_t k_max_cache_levels = 4; - // This is a string that identifies the current "version" of the hash sum // computed by ccache. If, for any reason, we want to force the hash sum to be // different for the same input in a new ccache version, we can just change // this string. A typical example would be if the format of one of the files // stored in the cache changes in a backwards-incompatible way. -const char HASH_PREFIX[] = "3"; +const char HASH_PREFIX[] = "4"; namespace { -// Throw a Failure if ccache did not succeed in getting or putting a result in -// the cache. If `exit_code` is set, just exit with that code directly, -// otherwise execute the real compiler and exit with its exit code. Also updates -// statistics counter `statistic` if it's not `Statistic::none`. -class Failure : public std::exception +// Return nonstd::make_unexpected if ccache did not succeed in getting +// or putting a result in the cache. If `exit_code` is set, ccache will just +// exit with that code directly, otherwise execute the real compiler and exit +// with its exit code. Statistics counters will also be incremented. +class Failure { public: - Failure(Statistic statistic, - nonstd::optional exit_code = nonstd::nullopt); + Failure(Statistic statistic); + Failure(std::initializer_list statistics); + const core::StatisticsCounters& counters() const; nonstd::optional exit_code() const; - Statistic statistic() const; + void set_exit_code(int exit_code); private: - Statistic m_statistic; + core::StatisticsCounters m_counters; nonstd::optional m_exit_code; }; -inline Failure::Failure(Statistic statistic, nonstd::optional exit_code) - : m_statistic(statistic), m_exit_code(exit_code) +inline Failure::Failure(const Statistic statistic) : m_counters({statistic}) +{ +} + +inline Failure::Failure(const std::initializer_list statistics) + : m_counters(statistics) { } +inline const core::StatisticsCounters& +Failure::counters() const +{ + return m_counters; +} + inline nonstd::optional Failure::exit_code() const { return m_exit_code; } -inline Statistic -Failure::statistic() const +inline void +Failure::set_exit_code(const int exit_code) { - return m_statistic; + m_exit_code = exit_code; } } // namespace @@ -234,7 +151,7 @@ for (const auto& word : Util::split_into_strings(prefix_command, " ")) { std::string path = find_executable(ctx, word, CCACHE_NAME); if (path.empty()) { - throw Fatal("{}: {}", word, strerror(errno)); + throw core::Fatal("{}: {}", word, strerror(errno)); } prefix.push_back(path); @@ -246,44 +163,15 @@ } } -static void -clean_up_internal_tempdir(const Config& config) -{ - time_t now = time(nullptr); - auto dir_st = Stat::stat(config.cache_dir(), Stat::OnError::log); - if (!dir_st || dir_st.mtime() + k_tempdir_cleanup_interval >= now) { - // No cleanup needed. - return; - } - - Util::update_mtime(config.cache_dir()); - - const std::string& temp_dir = config.temporary_dir(); - if (!Stat::lstat(temp_dir)) { - return; - } - - Util::traverse(temp_dir, [now](const std::string& path, bool is_dir) { - if (is_dir) { - return; - } - auto st = Stat::lstat(path, Stat::OnError::log); - if (st && st.mtime() + k_tempdir_cleanup_interval < now) { - Util::unlink_tmp(path); - } - }); -} - static std::string prepare_debug_path(const std::string& debug_dir, const std::string& output_obj, string_view suffix) { - const std::string prefix = - debug_dir.empty() ? output_obj : debug_dir + Util::real_path(output_obj); - try { - Util::ensure_dir_exists(Util::dir_name(prefix)); - } catch (Error&) { + auto prefix = debug_dir.empty() + ? output_obj + : debug_dir + util::to_absolute_path_no_drive(output_obj); + if (!Util::create_dir(Util::dir_name(prefix))) { // Ignore since we can't handle an error in another way in this context. The // caller takes care of logging when trying to open the path for writing. } @@ -325,7 +213,7 @@ if (symlink_value.empty()) { break; } - if (Util::is_absolute_path(symlink_value)) { + if (util::is_absolute_path(symlink_value)) { compiler_path = symlink_value; } else { compiler_path = @@ -357,14 +245,14 @@ // The comparison using >= is intentional, due to a possible race between // starting compilation and writing the include file. See also the notes under // "Performance" in doc/MANUAL.adoc. - if (!(ctx.config.sloppiness() & SLOPPY_INCLUDE_FILE_MTIME) + if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::include_file_mtime)) && path_stat.mtime() >= ctx.time_of_compilation) { LOG("Include file {} too new", path); return true; } // The same >= logic as above applies to the change time of the file. - if (!(ctx.config.sloppiness() & SLOPPY_INCLUDE_FILE_CTIME) + if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::include_file_ctime)) && path_stat.ctime() >= ctx.time_of_compilation) { LOG("Include file {} ctime too new", path); return true; @@ -393,21 +281,22 @@ return true; } - if (system && (ctx.config.sloppiness() & SLOPPY_SYSTEM_HEADERS)) { + if (system + && (ctx.config.sloppiness().is_enabled(core::Sloppy::system_headers))) { // Don't remember this system header. return true; } + // Canonicalize path for comparison; Clang uses ./header.h. + if (util::starts_with(path, "./")) { + path.erase(0, 2); + } + if (ctx.included_files.find(path) != ctx.included_files.end()) { // Already known include file. return true; } - // Canonicalize path for comparison; Clang uses ./header.h. - if (Util::starts_with(path, "./")) { - path.erase(0, 2); - } - #ifdef _WIN32 { // stat fails on directories on win32. @@ -461,7 +350,7 @@ Hash fhash; if (is_pch) { - if (ctx.included_pch_file.empty()) { + if (ctx.args_info.included_pch_file.empty()) { LOG("Detected use of precompiled header: {}", path); } bool using_pch_sum = false; @@ -542,10 +431,7 @@ // - Makes include file paths for which the base directory is a prefix relative // when computing the hash sum. // - Stores the paths and hashes of included files in ctx.included_files. -// -// Returns Statistic::none on success, otherwise a statistics counter to be -// incremented. -static Statistic +static nonstd::expected process_preprocessed_file(Context& ctx, Hash& hash, const std::string& path, @@ -554,8 +440,8 @@ std::string data; try { data = Util::read_file(path); - } catch (Error&) { - return Statistic::internal_error; + } catch (core::Error&) { + return nonstd::make_unexpected(Statistic::internal_error); } // Bytes between p and q are pending to be hashed. @@ -597,14 +483,14 @@ // GCC: && ((q[1] == ' ' && q[2] >= '0' && q[2] <= '9') // GCC precompiled header: - || Util::starts_with(&q[1], pragma_gcc_pch_preprocess) + || util::starts_with(&q[1], pragma_gcc_pch_preprocess) // HP/AIX: || (q[1] == 'l' && q[2] == 'i' && q[3] == 'n' && q[4] == 'e' && q[5] == ' ')) && (q == data.data() || q[-1] == '\n')) { // Workarounds for preprocessor linemarker bugs in GCC version 6. if (q[2] == '3') { - if (Util::starts_with(q, hash_31_command_line_newline)) { + if (util::starts_with(q, hash_31_command_line_newline)) { // Bogus extra line with #31, after the regular #1: Ignore the whole // line, and continue parsing. hash.hash(p, q - p); @@ -614,7 +500,7 @@ q++; p = q; continue; - } else if (Util::starts_with(q, hash_32_command_line_2_newline)) { + } else if (util::starts_with(q, hash_32_command_line_2_newline)) { // Bogus wrong line with #32, instead of regular #1: Replace the line // number with the usual one. hash.hash(p, q - p); @@ -636,7 +522,7 @@ q++; if (q >= end) { LOG_RAW("Failed to parse included file path"); - return Statistic::internal_error; + return nonstd::make_unexpected(Statistic::internal_error); } // q points to the beginning of an include file path hash.hash(p, q - p); @@ -656,14 +542,14 @@ // p and q span the include file path. std::string inc_path(p, q - p); if (!ctx.has_absolute_include_headers) { - ctx.has_absolute_include_headers = Util::is_absolute_path(inc_path); + ctx.has_absolute_include_headers = util::is_absolute_path(inc_path); } inc_path = Util::make_relative_path(ctx, inc_path); bool should_hash_inc_path = true; if (!ctx.config.hash_dir()) { - if (Util::starts_with(inc_path, ctx.apparent_cwd) - && Util::ends_with(inc_path, "//")) { + if (util::starts_with(inc_path, ctx.apparent_cwd) + && util::ends_with(inc_path, "//")) { // When compiling with -g or similar, GCC adds the absolute path to // CWD like this: // @@ -680,7 +566,8 @@ if (remember_include_file(ctx, inc_path, hash, system, nullptr) == RememberIncludeFileResult::cannot_use_pch) { - return Statistic::could_not_use_precompiled_header; + return nonstd::make_unexpected( + Statistic::could_not_use_precompiled_header); } p = q; // Everything of interest between p and q has been hashed now. } else if (q[0] == '.' && q[1] == 'i' && q[2] == 'n' && q[3] == 'c' @@ -692,7 +579,8 @@ LOG_RAW( "Found unsupported .inc" "bin directive in source code"); - throw Failure(Statistic::unsupported_code_directive); + return nonstd::make_unexpected( + Failure(Statistic::unsupported_code_directive)); } else if (pump && strncmp(q, "_________", 9) == 0) { // Unfortunately the distcc-pump wrapper outputs standard output lines: // __________Using distcc-pump from /usr/bin @@ -715,8 +603,9 @@ // Explicitly check the .gch/.pch/.pth file as Clang does not include any // mention of it in the preprocessed output. - if (!ctx.included_pch_file.empty()) { - std::string pch_path = Util::make_relative_path(ctx, ctx.included_pch_file); + if (!ctx.args_info.included_pch_file.empty()) { + std::string pch_path = + Util::make_relative_path(ctx, ctx.args_info.included_pch_file); hash.hash(pch_path); remember_include_file(ctx, pch_path, hash, false, nullptr); } @@ -726,18 +615,18 @@ print_included_files(ctx, stdout); } - return Statistic::none; + return {}; } // Extract the used includes from the dependency file. Note that we cannot // distinguish system headers from other includes here. static optional -result_name_from_depfile(Context& ctx, Hash& hash) +result_key_from_depfile(Context& ctx, Hash& hash) { std::string file_content; try { file_content = Util::read_file(ctx.args_info.output_dep); - } catch (const Error& e) { + } catch (const core::Error& e) { LOG( "Cannot open dependency file {}: {}", ctx.args_info.output_dep, e.what()); return nullopt; @@ -748,7 +637,7 @@ continue; } if (!ctx.has_absolute_include_headers) { - ctx.has_absolute_include_headers = Util::is_absolute_path(token); + ctx.has_absolute_include_headers = util::is_absolute_path(token); } std::string path = Util::make_relative_path(ctx, token); remember_include_file(ctx, path, hash, false, &hash); @@ -756,8 +645,9 @@ // Explicitly check the .gch/.pch/.pth file as it may not be mentioned in the // dependencies output. - if (!ctx.included_pch_file.empty()) { - std::string pch_path = Util::make_relative_path(ctx, ctx.included_pch_file); + if (!ctx.args_info.included_pch_file.empty()) { + std::string pch_path = + Util::make_relative_path(ctx, ctx.args_info.included_pch_file); hash.hash(pch_path); remember_include_file(ctx, pch_path, hash, false, nullptr); } @@ -771,7 +661,7 @@ } // Execute the compiler/preprocessor, with logic to retry without requesting // colored diagnostics messages if that fails. -static int +static nonstd::expected do_execute(Context& ctx, Args& args, TemporaryFile&& tmp_stdout, @@ -805,14 +695,14 @@ tmp_stdout.path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0600)); if (!tmp_stdout.fd) { LOG("Failed to truncate {}: {}", tmp_stdout.path, strerror(errno)); - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::internal_error); } tmp_stderr.fd = Fd(open( tmp_stderr.path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0600)); if (!tmp_stderr.fd) { LOG("Failed to truncate {}: {}", tmp_stderr.path, strerror(errno)); - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::internal_error); } ctx.diagnostics_color_failed = true; @@ -823,96 +713,36 @@ return status; } -struct LookUpCacheFileResult -{ - std::string path; - Stat stat; - uint8_t level; -}; - -static LookUpCacheFileResult -look_up_cache_file(const std::string& cache_dir, - const Digest& name, - nonstd::string_view suffix) -{ - const auto name_string = FMT("{}{}", name.to_string(), suffix); - - for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; - ++level) { - const auto path = Util::get_path_in_cache(cache_dir, level, name_string); - const auto stat = Stat::stat(path); - if (stat) { - return {path, stat, level}; - } - } - - const auto shallowest_path = - Util::get_path_in_cache(cache_dir, k_min_cache_levels, name_string); - return {shallowest_path, Stat(), k_min_cache_levels}; -} - // Create or update the manifest file. static void -update_manifest_file(Context& ctx) +update_manifest_file(Context& ctx, + const Digest& manifest_key, + const Digest& result_key) { - if (!ctx.config.direct_mode() || ctx.config.read_only() - || ctx.config.read_only_direct()) { + if (ctx.config.read_only() || ctx.config.read_only_direct()) { return; } - ASSERT(ctx.manifest_path()); - ASSERT(ctx.result_path()); - - MTR_BEGIN("manifest", "manifest_put"); + ASSERT(ctx.config.direct_mode()); - const auto old_stat = Stat::stat(*ctx.manifest_path()); + MTR_SCOPE("manifest", "manifest_put"); // See comment in get_file_hash_index for why saving of timestamps is forced // for precompiled headers. const bool save_timestamp = - (ctx.config.sloppiness() & SLOPPY_FILE_STAT_MATCHES) + (ctx.config.sloppiness().is_enabled(core::Sloppy::file_stat_matches)) || ctx.args_info.output_is_precompiled_header; - LOG("Adding result name to {}", *ctx.manifest_path()); - if (!Manifest::put(ctx.config, - *ctx.manifest_path(), - *ctx.result_name(), - ctx.included_files, - ctx.time_of_compilation, - save_timestamp)) { - LOG("Failed to add result name to {}", *ctx.manifest_path()); - } else { - const auto new_stat = Stat::stat(*ctx.manifest_path(), Stat::OnError::log); - ctx.manifest_counter_updates.increment( - Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(old_stat, new_stat)); - ctx.manifest_counter_updates.increment(Statistic::files_in_cache, - !old_stat && new_stat ? 1 : 0); - } - MTR_END("manifest", "manifest_put"); -} - -static void -create_cachedir_tag(const Context& ctx) -{ - constexpr char cachedir_tag[] = - "Signature: 8a477f597d28d172789f06886806bc55\n" - "# This file is a cache directory tag created by ccache.\n" - "# For information about cache directory tags, see:\n" - "#\thttp://www.brynosaurus.com/cachedir/\n"; - - const std::string path = FMT("{}/{}/CACHEDIR.TAG", - ctx.config.cache_dir(), - ctx.result_name()->to_string()[0]); - const auto stat = Stat::stat(path); - if (stat) { - return; - } - try { - Util::write_file(path, cachedir_tag); - } catch (const Error& e) { - LOG("Failed to create {}: {}", path, e.what()); - } + ctx.storage.put( + manifest_key, core::CacheEntryType::manifest, [&](const auto& path) { + LOG("Adding result key to {}", path); + return Manifest::put(ctx.config, + path, + result_key, + ctx.included_files, + ctx.time_of_compilation, + save_timestamp); + }); } struct FindCoverageFileResult @@ -953,10 +783,70 @@ return {true, found_file, found_file == mangled_form}; } -// Run the real compiler and put the result in cache. -static void +static bool +write_result(Context& ctx, + const std::string& result_path, + const Stat& obj_stat, + const std::string& stderr_path) +{ + Result::Writer result_writer(ctx, result_path); + + const auto stderr_stat = Stat::stat(stderr_path, Stat::OnError::log); + if (!stderr_stat) { + return false; + } + + if (stderr_stat.size() > 0) { + result_writer.write(Result::FileType::stderr_output, stderr_path); + } + if (obj_stat) { + result_writer.write(Result::FileType::object, ctx.args_info.output_obj); + } + if (ctx.args_info.generating_dependencies) { + result_writer.write(Result::FileType::dependency, ctx.args_info.output_dep); + } + if (ctx.args_info.generating_coverage) { + const auto coverage_file = find_coverage_file(ctx); + if (!coverage_file.found) { + return false; + } + result_writer.write(coverage_file.mangled + ? Result::FileType::coverage_mangled + : Result::FileType::coverage_unmangled, + coverage_file.path); + } + if (ctx.args_info.generating_stackusage) { + result_writer.write(Result::FileType::stackusage, ctx.args_info.output_su); + } + if (ctx.args_info.generating_diagnostics) { + result_writer.write(Result::FileType::diagnostic, ctx.args_info.output_dia); + } + if (ctx.args_info.seen_split_dwarf && Stat::stat(ctx.args_info.output_dwo)) { + // Only store .dwo file if it was created by the compiler (GCC and Clang + // behave differently e.g. for "-gsplit-dwarf -g1"). + result_writer.write(Result::FileType::dwarf_object, + ctx.args_info.output_dwo); + } + + const auto file_size_and_count_diff = result_writer.finalize(); + if (file_size_and_count_diff) { + ctx.storage.primary.increment_statistic( + Statistic::cache_size_kibibyte, file_size_and_count_diff->size_kibibyte); + ctx.storage.primary.increment_statistic(Statistic::files_in_cache, + file_size_and_count_diff->count); + } else { + LOG("Error: {}", file_size_and_count_diff.error()); + return false; + } + + return true; +} + +// Run the real compiler and put the result in cache. Returns the result key. +static nonstd::expected to_cache(Context& ctx, Args& args, + nonstd::optional result_key, const Args& depend_extra_args, Hash* depend_mode_hash) { @@ -996,7 +886,7 @@ if (unlink(ctx.args_info.output_dwo.c_str()) != 0 && errno != ENOENT && errno != ESTALE) { LOG("Failed to unlink {}: {}", ctx.args_info.output_dwo, strerror(errno)); - throw Failure(Statistic::bad_output_file); + return nonstd::make_unexpected(Statistic::bad_output_file); } } @@ -1011,7 +901,7 @@ ctx.register_pending_tmp_file(tmp_stderr.path); std::string tmp_stderr_path = tmp_stderr.path; - int status; + nonstd::expected status; if (!ctx.config.depend_mode()) { status = do_execute(ctx, args, std::move(tmp_stdout), std::move(tmp_stderr)); @@ -1030,17 +920,21 @@ } MTR_END("execute", "compiler"); + if (!status) { + return nonstd::make_unexpected(status.error()); + } + auto st = Stat::stat(tmp_stdout_path, Stat::OnError::log); if (!st) { // The stdout file was removed - cleanup in progress? Better bail out. - throw Failure(Statistic::missing_cache_file); + return nonstd::make_unexpected(Statistic::missing_cache_file); } // distcc-pump outputs lines like this: // __________Using # distcc servers in pump mode if (st.size() != 0 && ctx.config.compiler_type() != CompilerType::pump) { LOG_RAW("Compiler produced stdout"); - throw Failure(Statistic::compiler_produced_stdout); + return nonstd::make_unexpected(Statistic::compiler_produced_stdout); } // Merge stderr from the preprocessor (if any) and stderr from the real @@ -1052,23 +946,26 @@ } if (status != 0) { - LOG("Compiler gave exit status {}", status); + LOG("Compiler gave exit status {}", *status); // We can output stderr immediately instead of rerunning the compiler. Util::send_to_stderr(ctx, Util::read_file(tmp_stderr_path)); - throw Failure(Statistic::compile_failed, status); + auto failure = Failure(Statistic::compile_failed); + failure.set_exit_code(*status); + return nonstd::make_unexpected(failure); } if (ctx.config.depend_mode()) { ASSERT(depend_mode_hash); - auto result_name = result_name_from_depfile(ctx, *depend_mode_hash); - if (!result_name) { - throw Failure(Statistic::internal_error); + result_key = result_key_from_depfile(ctx, *depend_mode_hash); + if (!result_key) { + return nonstd::make_unexpected(Statistic::internal_error); } - ctx.set_result_name(*result_name); } + ASSERT(result_key); + bool produce_dep_file = ctx.args_info.generating_dependencies && ctx.args_info.output_dep != "/dev/null"; @@ -1080,100 +977,44 @@ if (!obj_stat) { if (ctx.args_info.expect_output_obj) { LOG_RAW("Compiler didn't produce an object file (unexpected)"); - throw Failure(Statistic::compiler_produced_no_output); + return nonstd::make_unexpected(Statistic::compiler_produced_no_output); } else { LOG_RAW("Compiler didn't produce an object file (expected)"); } } else if (obj_stat.size() == 0) { LOG_RAW("Compiler produced an empty object file"); - throw Failure(Statistic::compiler_produced_empty_output); - } - - const auto stderr_stat = Stat::stat(tmp_stderr_path, Stat::OnError::log); - if (!stderr_stat) { - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::compiler_produced_empty_output); } - const auto result_file = look_up_cache_file( - ctx.config.cache_dir(), *ctx.result_name(), Result::k_file_suffix); - ctx.set_result_path(result_file.path); - Result::Writer result_writer(ctx, result_file.path); - - if (stderr_stat.size() > 0) { - result_writer.write(Result::FileType::stderr_output, tmp_stderr_path); - } - if (obj_stat) { - result_writer.write(Result::FileType::object, ctx.args_info.output_obj); - } - if (ctx.args_info.generating_dependencies) { - result_writer.write(Result::FileType::dependency, ctx.args_info.output_dep); - } - if (ctx.args_info.generating_coverage) { - const auto coverage_file = find_coverage_file(ctx); - if (!coverage_file.found) { - throw Failure(Statistic::internal_error); - } - result_writer.write(coverage_file.mangled - ? Result::FileType::coverage_mangled - : Result::FileType::coverage_unmangled, - coverage_file.path); - } - if (ctx.args_info.generating_stackusage) { - result_writer.write(Result::FileType::stackusage, ctx.args_info.output_su); - } - if (ctx.args_info.generating_diagnostics) { - result_writer.write(Result::FileType::diagnostic, ctx.args_info.output_dia); - } - if (ctx.args_info.seen_split_dwarf && Stat::stat(ctx.args_info.output_dwo)) { - // Only store .dwo file if it was created by the compiler (GCC and Clang - // behave differently e.g. for "-gsplit-dwarf -g1"). - result_writer.write(Result::FileType::dwarf_object, - ctx.args_info.output_dwo); - } - - auto error = result_writer.finalize(); - if (error) { - LOG("Error: {}", *error); - } else { - LOG("Stored in cache: {}", result_file.path); - } - - auto new_result_stat = Stat::stat(result_file.path, Stat::OnError::log); - if (!new_result_stat) { - throw Failure(Statistic::internal_error); + MTR_BEGIN("result", "result_put"); + const bool added = ctx.storage.put( + *result_key, core::CacheEntryType::result, [&](const auto& path) { + return write_result(ctx, path, obj_stat, tmp_stderr_path); + }); + MTR_END("result", "result_put"); + if (!added) { + return nonstd::make_unexpected(Statistic::internal_error); } - ctx.counter_updates.increment( - Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(result_file.stat, new_result_stat)); - ctx.counter_updates.increment(Statistic::files_in_cache, - result_file.stat ? 0 : 1); - - MTR_END("file", "file_put"); - - // Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can - // be done almost anywhere, but we might as well do it near the end as we save - // the stat call if we exit early. - create_cachedir_tag(ctx); // Everything OK. Util::send_to_stderr(ctx, Util::read_file(tmp_stderr_path)); + + return *result_key; } -// Find the result name by running the compiler in preprocessor mode and +// Find the result key by running the compiler in preprocessor mode and // hashing the result. -static Digest -get_result_name_from_cpp(Context& ctx, Args& args, Hash& hash) +static nonstd::expected +get_result_key_from_cpp(Context& ctx, Args& args, Hash& hash) { ctx.time_of_compilation = time(nullptr); std::string stderr_path; std::string stdout_path; - int status; if (ctx.args_info.direct_i_file) { // We are compiling a .i or .ii file - that means we can skip the cpp stage // and directly form the correct i_tmpfile. stdout_path = ctx.args_info.input_file; - status = 0; } else { // Run cpp on the input file to obtain the .i. @@ -1207,30 +1048,28 @@ add_prefix(ctx, args, ctx.config.prefix_command_cpp()); LOG_RAW("Running preprocessor"); MTR_BEGIN("execute", "preprocessor"); - status = + const auto status = do_execute(ctx, args, std::move(tmp_stdout), std::move(tmp_stderr)); MTR_END("execute", "preprocessor"); args.pop_back(args_added); - } - if (status != 0) { - LOG("Preprocessor gave exit status {}", status); - throw Failure(Statistic::preprocessor_error); + if (!status) { + return nonstd::make_unexpected(status.error()); + } else if (*status != 0) { + LOG("Preprocessor gave exit status {}", *status); + return nonstd::make_unexpected(Statistic::preprocessor_error); + } } hash.hash_delimiter("cpp"); const bool is_pump = ctx.config.compiler_type() == CompilerType::pump; - const Statistic error = - process_preprocessed_file(ctx, hash, stdout_path, is_pump); - if (error != Statistic::none) { - throw Failure(error); - } + TRY(process_preprocessed_file(ctx, hash, stdout_path, is_pump)); hash.hash_delimiter("cppstderr"); if (!ctx.args_info.direct_i_file && !hash.hash_file(stderr_path)) { // Somebody removed the temporary file? LOG("Failed to open {}: {}", stderr_path, strerror(errno)); - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::internal_error); } if (ctx.args_info.direct_i_file) { @@ -1252,7 +1091,7 @@ // Hash mtime or content of a file, or the output of a command, according to // the CCACHE_COMPILERCHECK setting. -static void +static nonstd::expected hash_compiler(const Context& ctx, Hash& hash, const Stat& st, @@ -1265,7 +1104,7 @@ hash.hash_delimiter("cc_mtime"); hash.hash(st.size()); hash.hash(st.mtime()); - } else if (Util::starts_with(ctx.config.compiler_check(), "string:")) { + } else if (util::starts_with(ctx.config.compiler_check(), "string:")) { hash.hash_delimiter("cc_hash"); hash.hash(&ctx.config.compiler_check()[7]); } else if (ctx.config.compiler_check() == "content" || !allow_command) { @@ -1276,9 +1115,10 @@ hash, ctx.config.compiler_check(), ctx.orig_args[0])) { LOG("Failure running compiler check command: {}", ctx.config.compiler_check()); - throw Failure(Statistic::compiler_check_failed); + return nonstd::make_unexpected(Statistic::compiler_check_failed); } } + return {}; } // Hash the host compiler(s) invoked by nvcc. @@ -1286,7 +1126,7 @@ // If `ccbin_st` and `ccbin` are set, they refer to a directory or compiler set // with -ccbin/--compiler-bindir. If `ccbin_st` is nullptr or `ccbin` is the // empty string, the compilers are looked up in PATH instead. -static void +static nonstd::expected hash_nvcc_host_compiler(const Context& ctx, Hash& hash, const Stat* ccbin_st = nullptr, @@ -1317,19 +1157,21 @@ std::string path = FMT("{}/{}", ccbin, compiler); auto st = Stat::stat(path); if (st) { - hash_compiler(ctx, hash, st, path, false); + TRY(hash_compiler(ctx, hash, st, path, false)); } } else { std::string path = find_executable(ctx, compiler, CCACHE_NAME); if (!path.empty()) { auto st = Stat::stat(path, Stat::OnError::log); - hash_compiler(ctx, hash, st, ccbin, false); + TRY(hash_compiler(ctx, hash, st, ccbin, false)); } } } } else { - hash_compiler(ctx, hash, *ccbin_st, ccbin, false); + TRY(hash_compiler(ctx, hash, *ccbin_st, ccbin, false)); } + + return {}; } static bool @@ -1339,7 +1181,7 @@ } // update a hash with information common for the direct and preprocessor modes. -static void +static nonstd::expected hash_common_info(const Context& ctx, const Args& args, Hash& hash, @@ -1347,6 +1189,11 @@ { hash.hash(HASH_PREFIX); + if (!ctx.config.namespace_().empty()) { + hash.hash_delimiter("namespace"); + hash.hash(ctx.config.namespace_()); + } + // We have to hash the extension, as a .i file isn't treated the same by the // compiler as a .ii file. hash.hash_delimiter("ext"); @@ -1360,11 +1207,11 @@ auto st = Stat::stat(compiler_path, Stat::OnError::log); if (!st) { - throw Failure(Statistic::could_not_find_compiler); + return nonstd::make_unexpected(Statistic::could_not_find_compiler); } // Hash information about the compiler. - hash_compiler(ctx, hash, st, compiler_path, true); + TRY(hash_compiler(ctx, hash, st, compiler_path, true)); // Also hash the compiler name as some compilers use hard links and behave // differently depending on the real name. @@ -1387,7 +1234,7 @@ } } - if (!(ctx.config.sloppiness() & SLOPPY_LOCALE)) { + if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::locale))) { // Hash environment variables that may affect localization of compiler // warning messages. const char* envvars[] = { @@ -1413,7 +1260,7 @@ old_path, new_path, ctx.apparent_cwd); - if (Util::starts_with(ctx.apparent_cwd, old_path)) { + if (util::starts_with(ctx.apparent_cwd, old_path)) { dir_to_hash = new_path + ctx.apparent_cwd.substr(old_path.size()); } } @@ -1425,14 +1272,17 @@ if ((!should_rewrite_dependency_target(ctx.args_info) && ctx.args_info.generating_dependencies) - || ctx.args_info.seen_split_dwarf) { - // The output object file name is part of the .d file, so include the path - // in the hash if generating dependencies. + || ctx.args_info.seen_split_dwarf || ctx.args_info.profile_arcs) { + // If generating dependencies: The output object file name is part of the .d + // file, so include the path in the hash. // - // Object files include a link to the corresponding .dwo file based on the - // target object filename when using -gsplit-dwarf, so hashing the object - // file path will do it, although just hashing the object file base name - // would be enough. + // When using -gsplit-dwarf: Object files include a link to the + // corresponding .dwo file based on the target object filename, so hashing + // the object file path will do it, although just hashing the object file + // base name would be enough. + // + // When using -fprofile-arcs (including implicitly via --coverage): the + // object file contains a .gcda path based on the object file path. hash.hash_delimiter("object file"); hash.hash(ctx.args_info.output_obj); } @@ -1459,17 +1309,17 @@ LOG("Hashing sanitize blacklist {}", sanitize_blacklist); hash.hash("sanitizeblacklist"); if (!hash_binary_file(ctx, hash, sanitize_blacklist)) { - throw Failure(Statistic::error_hashing_extra_file); + return nonstd::make_unexpected(Statistic::error_hashing_extra_file); } } if (!ctx.config.extra_files_to_hash().empty()) { - for (const std::string& path : Util::split_into_strings( - ctx.config.extra_files_to_hash(), PATH_DELIM)) { + for (const std::string& path : + util::split_path_list(ctx.config.extra_files_to_hash())) { LOG("Hashing extra file {}", path); hash.hash_delimiter("extrafile"); if (!hash_binary_file(ctx, hash, path)) { - throw Failure(Statistic::error_hashing_extra_file); + return nonstd::make_unexpected(Statistic::error_hashing_extra_file); } } } @@ -1482,6 +1332,8 @@ hash.hash(gcc_colors); } } + + return {}; } static bool @@ -1526,23 +1378,25 @@ const std::vector& patterns) { return std::any_of( - patterns.cbegin(), patterns.cend(), [&arg](const std::string& pattern) { + patterns.cbegin(), patterns.cend(), [&arg](const auto& pattern) { const auto& prefix = string_view(pattern).substr(0, pattern.length() - 1); return ( pattern == arg - || (Util::ends_with(pattern, "*") && Util::starts_with(arg, prefix))); + || (util::ends_with(pattern, "*") && util::starts_with(arg, prefix))); }); } // Update a hash sum with information specific to the direct and preprocessor -// modes and calculate the result name. Returns the result name on success, -// otherwise nullopt. -static optional -calculate_result_name(Context& ctx, - const Args& args, - Args& preprocessor_args, - Hash& hash, - bool direct_mode) +// modes and calculate the result key. Returns the result key on success, and +// if direct_mode is true also the manifest key. +static nonstd::expected< + std::pair, nonstd::optional>, + Failure> +calculate_result_and_manifest_key(Context& ctx, + const Args& args, + Args& preprocessor_args, + Hash& hash, + bool direct_mode) { bool found_ccbin = false; @@ -1576,12 +1430,12 @@ i++; continue; } - if (Util::starts_with(args[i], "-L") && !is_clang) { + if (util::starts_with(args[i], "-L") && !is_clang) { continue; } // -Wl,... doesn't affect compilation (except for clang). - if (Util::starts_with(args[i], "-Wl,") && !is_clang) { + if (util::starts_with(args[i], "-Wl,") && !is_clang) { continue; } @@ -1589,17 +1443,17 @@ // CCACHE_BASEDIR to reuse results across different directories. Skip using // the value of the option from hashing but still hash the existence of the // option. - if (Util::starts_with(args[i], "-fdebug-prefix-map=")) { + if (util::starts_with(args[i], "-fdebug-prefix-map=")) { hash.hash_delimiter("arg"); hash.hash("-fdebug-prefix-map="); continue; } - if (Util::starts_with(args[i], "-ffile-prefix-map=")) { + if (util::starts_with(args[i], "-ffile-prefix-map=")) { hash.hash_delimiter("arg"); hash.hash("-ffile-prefix-map="); continue; } - if (Util::starts_with(args[i], "-fmacro-prefix-map=")) { + if (util::starts_with(args[i], "-fmacro-prefix-map=")) { hash.hash_delimiter("arg"); hash.hash("-fmacro-prefix-map="); continue; @@ -1625,17 +1479,17 @@ // If we're generating dependencies, we make sure to skip the filename of // the dependency file, since it doesn't impact the output. if (ctx.args_info.generating_dependencies) { - if (Util::starts_with(args[i], "-Wp,")) { - if (Util::starts_with(args[i], "-Wp,-MD,") + if (util::starts_with(args[i], "-Wp,")) { + if (util::starts_with(args[i], "-Wp,-MD,") && args[i].find(',', 8) == std::string::npos) { hash.hash(args[i].data(), 8); continue; - } else if (Util::starts_with(args[i], "-Wp,-MMD,") + } else if (util::starts_with(args[i], "-Wp,-MMD,") && args[i].find(',', 9) == std::string::npos) { hash.hash(args[i].data(), 9); continue; } - } else if (Util::starts_with(args[i], "-MF")) { + } else if (util::starts_with(args[i], "-MF")) { // In either case, hash the "-MF" part. hash.hash_delimiter("arg"); hash.hash(args[i].data(), 3); @@ -1651,24 +1505,37 @@ } } - if (Util::starts_with(args[i], "-specs=") - || Util::starts_with(args[i], "--specs=")) { - std::string path = args[i].substr(args[i].find('=') + 1); + if (util::starts_with(args[i], "-specs=") + || util::starts_with(args[i], "--specs=") + || (args[i] == "-specs" || args[i] == "--specs") + || args[i] == "--config") { + std::string path; + size_t eq_pos = args[i].find('='); + if (eq_pos == std::string::npos) { + if (i + 1 >= args.size()) { + LOG("missing argument for \"{}\"", args[i]); + return nonstd::make_unexpected(Statistic::bad_compiler_arguments); + } + path = args[i + 1]; + i++; + } else { + path = args[i].substr(eq_pos + 1); + } auto st = Stat::stat(path, Stat::OnError::log); if (st) { // If given an explicit specs file, then hash that file, but don't // include the path to it in the hash. hash.hash_delimiter("specs"); - hash_compiler(ctx, hash, st, path, false); + TRY(hash_compiler(ctx, hash, st, path, false)); continue; } } - if (Util::starts_with(args[i], "-fplugin=")) { + if (util::starts_with(args[i], "-fplugin=")) { auto st = Stat::stat(&args[i][9], Stat::OnError::log); if (st) { hash.hash_delimiter("plugin"); - hash_compiler(ctx, hash, st, &args[i][9], false); + TRY(hash_compiler(ctx, hash, st, &args[i][9], false)); continue; } } @@ -1678,7 +1545,7 @@ auto st = Stat::stat(args[i + 3], Stat::OnError::log); if (st) { hash.hash_delimiter("plugin"); - hash_compiler(ctx, hash, st, args[i + 3], false); + TRY(hash_compiler(ctx, hash, st, args[i + 3], false)); i += 3; continue; } @@ -1690,7 +1557,7 @@ if (st) { found_ccbin = true; hash.hash_delimiter("ccbin"); - hash_nvcc_host_compiler(ctx, hash, &st, args[i + 1]); + TRY(hash_nvcc_host_compiler(ctx, hash, &st, args[i + 1])); i++; continue; } @@ -1714,7 +1581,7 @@ } if (!found_ccbin && ctx.args_info.actual_language == "cu") { - hash_nvcc_host_compiler(ctx, hash); + TRY(hash_nvcc_host_compiler(ctx, hash)); } // For profile generation (-fprofile(-instr)-generate[=path]) @@ -1732,14 +1599,22 @@ if (ctx.args_info.profile_generate) { ASSERT(!ctx.args_info.profile_path.empty()); - LOG("Adding profile directory {} to our hash", ctx.args_info.profile_path); + + // For a relative profile directory D the compiler stores $PWD/D as part of + // the profile filename so we need to include the same information in the + // hash. + const std::string profile_path = + util::is_absolute_path(ctx.args_info.profile_path) + ? ctx.args_info.profile_path + : FMT("{}/{}", ctx.apparent_cwd, ctx.args_info.profile_path); + LOG("Adding profile directory {} to our hash", profile_path); hash.hash_delimiter("-fprofile-dir"); - hash.hash(ctx.args_info.profile_path); + hash.hash(profile_path); } if (ctx.args_info.profile_use && !hash_profile_data_file(ctx, hash)) { LOG_RAW("No profile data file found"); - throw Failure(Statistic::no_input_file); + return nonstd::make_unexpected(Statistic::no_input_file); } // Adding -arch to hash since cpp output is affected. @@ -1748,7 +1623,9 @@ hash.hash(arch); } - optional result_name; + nonstd::optional result_key; + nonstd::optional manifest_key; + if (direct_mode) { // Hash environment variables that affect the preprocessor output. const char* envvars[] = {"CPATH", @@ -1782,68 +1659,70 @@ hash.hash_delimiter("sourcecode"); int result = hash_source_code_file(ctx, hash, ctx.args_info.input_file); if (result & HASH_SOURCE_CODE_ERROR) { - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::internal_error); } if (result & HASH_SOURCE_CODE_FOUND_TIME) { LOG_RAW("Disabling direct mode"); ctx.config.set_direct_mode(false); - return nullopt; + return std::make_pair(nullopt, nullopt); } - const auto manifest_name = hash.digest(); - ctx.set_manifest_name(manifest_name); + manifest_key = hash.digest(); - const auto manifest_file = look_up_cache_file( - ctx.config.cache_dir(), manifest_name, Manifest::k_file_suffix); - ctx.set_manifest_path(manifest_file.path); + const auto manifest_path = + ctx.storage.get(*manifest_key, core::CacheEntryType::manifest); - if (manifest_file.stat) { - LOG("Looking for result name in {}", manifest_file.path); + if (manifest_path) { + LOG("Looking for result key in {}", *manifest_path); MTR_BEGIN("manifest", "manifest_get"); - result_name = Manifest::get(ctx, manifest_file.path); + result_key = Manifest::get(ctx, *manifest_path); MTR_END("manifest", "manifest_get"); - if (result_name) { - LOG_RAW("Got result name from manifest"); + if (result_key) { + LOG_RAW("Got result key from manifest"); } else { - LOG_RAW("Did not find result name in manifest"); + LOG_RAW("Did not find result key in manifest"); } - } else { - LOG("No manifest with name {} in the cache", manifest_name.to_string()); } + } else if (ctx.args_info.arch_args.empty()) { + const auto digest = get_result_key_from_cpp(ctx, preprocessor_args, hash); + if (!digest) { + return nonstd::make_unexpected(digest.error()); + } + result_key = *digest; + LOG_RAW("Got result key from preprocessor"); } else { - if (ctx.args_info.arch_args.empty()) { - result_name = get_result_name_from_cpp(ctx, preprocessor_args, hash); - LOG_RAW("Got result name from preprocessor"); - } else { - preprocessor_args.push_back("-arch"); - for (size_t i = 0; i < ctx.args_info.arch_args.size(); ++i) { - preprocessor_args.push_back(ctx.args_info.arch_args[i]); - result_name = get_result_name_from_cpp(ctx, preprocessor_args, hash); - LOG("Got result name from preprocessor with -arch {}", - ctx.args_info.arch_args[i]); - if (i != ctx.args_info.arch_args.size() - 1) { - result_name = nullopt; - } - preprocessor_args.pop_back(); + preprocessor_args.push_back("-arch"); + for (size_t i = 0; i < ctx.args_info.arch_args.size(); ++i) { + preprocessor_args.push_back(ctx.args_info.arch_args[i]); + const auto digest = get_result_key_from_cpp(ctx, preprocessor_args, hash); + if (!digest) { + return nonstd::make_unexpected(digest.error()); + } + result_key = *digest; + LOG("Got result key from preprocessor with -arch {}", + ctx.args_info.arch_args[i]); + if (i != ctx.args_info.arch_args.size() - 1) { + result_key = nullopt; } preprocessor_args.pop_back(); } + preprocessor_args.pop_back(); } - return result_name; + return std::make_pair(result_key, manifest_key); } enum class FromCacheCallMode { direct, cpp }; // Try to return the compile result from cache. -static optional -from_cache(Context& ctx, FromCacheCallMode mode) +static bool +from_cache(Context& ctx, FromCacheCallMode mode, const Digest& result_key) { UmaskScope umask_scope(ctx.original_umask); // The user might be disabling cache hits. if (ctx.config.recache()) { - return nullopt; + return false; } // If we're using Clang, we can't trust a precompiled header object based on @@ -1856,39 +1735,32 @@ if ((ctx.config.compiler_type() == CompilerType::clang || ctx.config.compiler_type() == CompilerType::other) && ctx.args_info.output_is_precompiled_header - && !ctx.args_info.fno_pch_timestamp && mode == FromCacheCallMode::cpp) { + && mode == FromCacheCallMode::cpp) { LOG_RAW("Not considering cached precompiled header in preprocessor mode"); - return nullopt; + return false; } - MTR_BEGIN("cache", "from_cache"); + MTR_SCOPE("cache", "from_cache"); // Get result from cache. - const auto result_file = look_up_cache_file( - ctx.config.cache_dir(), *ctx.result_name(), Result::k_file_suffix); - if (!result_file.stat) { - LOG("No result with name {} in the cache", ctx.result_name()->to_string()); - return nullopt; + const auto result_path = + ctx.storage.get(result_key, core::CacheEntryType::result); + if (!result_path) { + return false; } - ctx.set_result_path(result_file.path); - Result::Reader result_reader(result_file.path); + + Result::Reader result_reader(*result_path); ResultRetriever result_retriever( ctx, should_rewrite_dependency_target(ctx.args_info)); auto error = result_reader.read(result_retriever); - MTR_END("cache", "from_cache"); if (error) { LOG("Failed to get result from cache: {}", *error); - return nullopt; + return false; } - // Update modification timestamp to save file from LRU cleanup. - Util::update_mtime(*ctx.result_path()); - LOG_RAW("Succeeded getting cached result"); - - return mode == FromCacheCallMode::direct ? Statistic::direct_cache_hit - : Statistic::preprocessed_cache_hit; + return true; } // Find the real compiler and put it into ctx.orig_args[0]. We just search the @@ -1918,17 +1790,17 @@ : ctx.orig_args[compiler_pos]); const std::string resolved_compiler = - Util::is_full_path(compiler) + util::is_full_path(compiler) ? compiler : find_executable_function(ctx, compiler, CCACHE_NAME); if (resolved_compiler.empty()) { - throw Fatal("Could not find compiler \"{}\" in PATH", compiler); + throw core::Fatal("Could not find compiler \"{}\" in PATH", compiler); } if (Util::same_program_name(Util::base_name(resolved_compiler), CCACHE_NAME)) { - throw Fatal( + throw core::Fatal( "Recursive invocation (the name of the ccache binary must be \"{}\")", CCACHE_NAME); } @@ -1937,129 +1809,12 @@ ctx.orig_args[0] = resolved_compiler; } -static std::string -default_cache_dir(const std::string& home_dir) -{ -#ifdef _WIN32 - return home_dir + "/ccache"; -#elif defined(__APPLE__) - return home_dir + "/Library/Caches/ccache"; -#else - return home_dir + "/.cache/ccache"; -#endif -} - -static std::string -default_config_dir(const std::string& home_dir) -{ -#ifdef _WIN32 - return home_dir + "/ccache"; -#elif defined(__APPLE__) - return home_dir + "/Library/Preferences/ccache"; -#else - return home_dir + "/.config/ccache"; -#endif -} - -// Read config file(s), populate variables, create configuration file in cache -// directory if missing, etc. -static void -set_up_config(Config& config) -{ - const std::string home_dir = Util::get_home_directory(); - const std::string legacy_ccache_dir = home_dir + "/.ccache"; - const bool legacy_ccache_dir_exists = - Stat::stat(legacy_ccache_dir).is_directory(); - const char* const env_xdg_cache_home = getenv("XDG_CACHE_HOME"); - const char* const env_xdg_config_home = getenv("XDG_CONFIG_HOME"); - - const char* env_ccache_configpath = getenv("CCACHE_CONFIGPATH"); - if (env_ccache_configpath) { - config.set_primary_config_path(env_ccache_configpath); - } else { - // Only used for ccache tests: - const char* const env_ccache_configpath2 = getenv("CCACHE_CONFIGPATH2"); - - config.set_secondary_config_path(env_ccache_configpath2 - ? env_ccache_configpath2 - : FMT("{}/ccache.conf", SYSCONFDIR)); - MTR_BEGIN("config", "conf_read_secondary"); - // A missing config file in SYSCONFDIR is OK so don't check return value. - config.update_from_file(config.secondary_config_path()); - MTR_END("config", "conf_read_secondary"); - - const char* const env_ccache_dir = getenv("CCACHE_DIR"); - std::string primary_config_dir; - if (env_ccache_dir && *env_ccache_dir) { - primary_config_dir = env_ccache_dir; - } else if (!config.cache_dir().empty() && !env_ccache_dir) { - primary_config_dir = config.cache_dir(); - } else if (legacy_ccache_dir_exists) { - primary_config_dir = legacy_ccache_dir; - } else if (env_xdg_config_home) { - primary_config_dir = FMT("{}/ccache", env_xdg_config_home); - } else { - primary_config_dir = default_config_dir(home_dir); - } - config.set_primary_config_path(primary_config_dir + "/ccache.conf"); - } - - const std::string& cache_dir_before_primary_config = config.cache_dir(); - - MTR_BEGIN("config", "conf_read_primary"); - config.update_from_file(config.primary_config_path()); - MTR_END("config", "conf_read_primary"); - - // Ignore cache_dir set in primary config. - config.set_cache_dir(cache_dir_before_primary_config); - - MTR_BEGIN("config", "conf_update_from_environment"); - config.update_from_environment(); - // (config.cache_dir is set above if CCACHE_DIR is set.) - MTR_END("config", "conf_update_from_environment"); - - if (config.cache_dir().empty()) { - if (legacy_ccache_dir_exists) { - config.set_cache_dir(legacy_ccache_dir); - } else if (env_xdg_cache_home) { - config.set_cache_dir(FMT("{}/ccache", env_xdg_cache_home)); - } else { - config.set_cache_dir(default_cache_dir(home_dir)); - } - } - // else: cache_dir was set explicitly via environment or via secondary config. - - // We have now determined config.cache_dir and populated the rest of config in - // prio order (1. environment, 2. primary config, 3. secondary config). -} - -static void -set_up_context(Context& ctx, int argc, const char* const* argv) -{ - ctx.orig_args = Args::from_argv(argc, argv); - ctx.ignore_header_paths = Util::split_into_strings( - ctx.config.ignore_headers_in_manifest(), PATH_DELIM); - ctx.set_ignore_options( - Util::split_into_strings(ctx.config.ignore_options(), " ")); -} - -// Initialize ccache, must be called once before anything else is run. +// Initialize ccache. Must be called once before anything else is run. static void initialize(Context& ctx, int argc, const char* const* argv) { - set_up_config(ctx.config); - set_up_context(ctx, argc, argv); - Logging::init(ctx.config); - - // Set default umask for all files created by ccache from now on (if - // configured to). This is intentionally done after calling init_log so that - // the log file won't be affected by the umask but before creating the initial - // configuration file. The intention is that all files and directories in the - // cache directory should be affected by the configured umask and that no - // other files and directories should. - if (ctx.config.umask() != std::numeric_limits::max()) { - ctx.original_umask = umask(ctx.config.umask()); - } + ctx.orig_args = Args::from_argv(argc, argv); + ctx.storage.initialize(); LOG("=== CCACHE {} STARTED =========================================", CCACHE_VERSION); @@ -2075,197 +1830,70 @@ // Make a copy of stderr that will not be cached, so things like distcc can // send networking errors to it. -static void +static nonstd::expected set_up_uncached_err() { int uncached_fd = dup(STDERR_FILENO); // The file descriptor is intentionally leaked. if (uncached_fd == -1) { LOG("dup(2) failed: {}", strerror(errno)); - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::internal_error); } Util::setenv("UNCACHED_ERR_FD", FMT("{}", uncached_fd)); -} - -static void -configuration_logger(const std::string& key, - const std::string& value, - const std::string& origin) -{ - BULK_LOG("Config: ({}) {} = {}", origin, key, value); -} - -static void -configuration_printer(const std::string& key, - const std::string& value, - const std::string& origin) -{ - PRINT(stdout, "({}) {} = {}\n", origin, key, value); + return {}; } static int cache_compilation(int argc, const char* const* argv); -static Statistic do_cache_compilation(Context& ctx, const char* const* argv); -static uint8_t -calculate_wanted_cache_level(uint64_t files_in_level_1) -{ - uint64_t files_per_directory = files_in_level_1 / 16; - for (uint8_t i = k_min_cache_levels; i <= k_max_cache_levels; ++i) { - if (files_per_directory < k_max_cache_files_per_directory) { - return i; - } - files_per_directory /= 16; - } - return k_max_cache_levels; -} +static nonstd::expected +do_cache_compilation(Context& ctx, const char* const* argv); -static optional -update_stats_and_maybe_move_cache_file(const Context& ctx, - const Digest& name, - const std::string& current_path, - const Counters& counter_updates, - const std::string& file_suffix) +static void +log_result_to_debug_log(Context& ctx) { - if (counter_updates.all_zero()) { - return nullopt; - } - - // Use stats file in the level one subdirectory for cache bookkeeping counters - // since cleanup is performed on level one. Use stats file in the level two - // subdirectory for other counters to reduce lock contention. - const bool use_stats_on_level_1 = - counter_updates.get(Statistic::cache_size_kibibyte) != 0 - || counter_updates.get(Statistic::files_in_cache) != 0; - std::string level_string = FMT("{:x}", name.bytes()[0] >> 4); - if (!use_stats_on_level_1) { - level_string += FMT("/{:x}", name.bytes()[0] & 0xF); - } - const auto stats_file = - FMT("{}/{}/stats", ctx.config.cache_dir(), level_string); - - auto counters = - Statistics::update(stats_file, [&counter_updates](Counters& cs) { - cs.increment(counter_updates); - }); - if (!counters) { - return nullopt; + if (ctx.config.log_file().empty() && !ctx.config.debug()) { + return; } - if (use_stats_on_level_1) { - // Only consider moving the cache file to another level when we have read - // the level 1 stats file since it's only then we know the proper - // files_in_cache value. - const auto wanted_level = - calculate_wanted_cache_level(counters->get(Statistic::files_in_cache)); - const auto wanted_path = Util::get_path_in_cache( - ctx.config.cache_dir(), wanted_level, name.to_string() + file_suffix); - if (current_path != wanted_path) { - Util::ensure_dir_exists(Util::dir_name(wanted_path)); - LOG("Moving {} to {}", current_path, wanted_path); - try { - Util::rename(current_path, wanted_path); - } catch (const Error&) { - // Two ccache processes may move the file at the same time, so failure - // to rename is OK. - } - } + core::Statistics statistics(ctx.storage.primary.get_statistics_updates()); + for (const auto& message : statistics.get_statistics_ids()) { + LOG("Result: {}", message); } - return counters; } static void -finalize_stats_and_trigger_cleanup(Context& ctx) +log_result_to_stats_log(Context& ctx) { - const auto& config = ctx.config; - - if (config.disable()) { - // Just log result, don't update statistics. - LOG_RAW("Result: disabled"); + if (ctx.config.stats_log().empty()) { return; } - if (!config.log_file().empty() || config.debug()) { - const auto result = Statistics::get_result(ctx.counter_updates); - if (result) { - LOG("Result: {}", *result); - } - } - - if (!config.stats()) { + core::Statistics statistics(ctx.storage.primary.get_statistics_updates()); + const auto ids = statistics.get_statistics_ids(); + if (ids.empty()) { return; } - if (!ctx.result_path()) { - ASSERT(ctx.counter_updates.get(Statistic::cache_size_kibibyte) == 0); - ASSERT(ctx.counter_updates.get(Statistic::files_in_cache) == 0); - - // Context::set_result_path hasn't been called yet, so we just choose one of - // the stats files in the 256 level 2 directories. - const auto bucket = getpid() % 256; - const auto stats_file = - FMT("{}/{:x}/{:x}/stats", config.cache_dir(), bucket / 16, bucket % 16); - Statistics::update( - stats_file, [&ctx](Counters& cs) { cs.increment(ctx.counter_updates); }); - return; - } - - if (ctx.manifest_path()) { - update_stats_and_maybe_move_cache_file(ctx, - *ctx.manifest_name(), - *ctx.manifest_path(), - ctx.manifest_counter_updates, - Manifest::k_file_suffix); - } - - const auto counters = - update_stats_and_maybe_move_cache_file(ctx, - *ctx.result_name(), - *ctx.result_path(), - ctx.counter_updates, - Result::k_file_suffix); - if (!counters) { - return; - } - - const auto subdir = - FMT("{}/{:x}", config.cache_dir(), ctx.result_name()->bytes()[0] >> 4); - bool need_cleanup = false; - - if (config.max_files() != 0 - && counters->get(Statistic::files_in_cache) > config.max_files() / 16) { - LOG("Need to clean up {} since it holds {} files (limit: {} files)", - subdir, - counters->get(Statistic::files_in_cache), - config.max_files() / 16); - need_cleanup = true; - } - if (config.max_size() != 0 - && counters->get(Statistic::cache_size_kibibyte) - > config.max_size() / 1024 / 16) { - LOG("Need to clean up {} since it holds {} KiB (limit: {} KiB)", - subdir, - counters->get(Statistic::cache_size_kibibyte), - config.max_size() / 1024 / 16); - need_cleanup = true; - } - - if (need_cleanup) { - const double factor = config.limit_multiple() / 16; - const uint64_t max_size = round(config.max_size() * factor); - const uint32_t max_files = round(config.max_files() * factor); - const time_t max_age = 0; - clean_up_dir( - subdir, max_size, max_files, max_age, [](double /*progress*/) {}); - } + core::StatsLog(ctx.config.stats_log()) + .log_result(ctx.args_info.input_file, ids); } static void finalize_at_exit(Context& ctx) { try { - finalize_stats_and_trigger_cleanup(ctx); - } catch (const ErrorBase& e) { + if (ctx.config.disable()) { + // Just log result, don't update statistics. + LOG_RAW("Result: disabled"); + return; + } + + log_result_to_debug_log(ctx); + log_result_to_stats_log(ctx); + + ctx.storage.finalize(); + } catch (const core::ErrorBase& e) { // finalize_at_exit must not throw since it's called by a destructor. LOG("Error while finalizing stats: {}", e.what()); } @@ -2285,11 +1913,12 @@ bool fall_back_to_original_compiler = false; Args saved_orig_args; - nonstd::optional original_umask; + nonstd::optional original_umask; std::string saved_temp_dir; { Context ctx; + ctx.initialize(); SignalHandler signal_handler(ctx); Finalizer finalizer([&ctx] { finalize_at_exit(ctx); }); @@ -2299,16 +1928,12 @@ find_compiler(ctx, &find_executable); MTR_END("main", "find_compiler"); - try { - Statistic statistic = do_cache_compilation(ctx, argv); - ctx.counter_updates.increment(statistic); - } catch (const Failure& e) { - if (e.statistic() != Statistic::none) { - ctx.counter_updates.increment(e.statistic()); - } - - if (e.exit_code()) { - return *e.exit_code(); + const auto result = do_cache_compilation(ctx, argv); + const auto& counters = result ? *result : result.error().counters(); + ctx.storage.primary.increment_statistics(counters); + if (!result) { + if (result.error().exit_code()) { + return *result.error().exit_code(); } // Else: Fall back to running the real compiler. fall_back_to_original_compiler = true; @@ -2337,33 +1962,36 @@ } auto execv_argv = saved_orig_args.to_argv(); execute_noreturn(execv_argv.data(), saved_temp_dir); - throw Fatal( + throw core::Fatal( "execute_noreturn of {} failed: {}", execv_argv[0], strerror(errno)); } return EXIT_SUCCESS; } -static Statistic +static nonstd::expected do_cache_compilation(Context& ctx, const char* const* argv) { if (ctx.actual_cwd.empty()) { LOG("Unable to determine current working directory: {}", strerror(errno)); - throw Failure(Statistic::internal_error); - } - - MTR_BEGIN("main", "clean_up_internal_tempdir"); - if (ctx.config.temporary_dir() == ctx.config.cache_dir() + "/tmp") { - clean_up_internal_tempdir(ctx.config); + return nonstd::make_unexpected(Statistic::internal_error); } - MTR_END("main", "clean_up_internal_tempdir"); if (!ctx.config.log_file().empty() || ctx.config.debug()) { - ctx.config.visit_items(configuration_logger); + ctx.config.visit_items([&ctx](const std::string& key, + const std::string& value, + const std::string& origin) { + const auto& log_value = + key == "secondary_storage" + ? ctx.storage.get_secondary_storage_config_for_logging() + : value; + BULK_LOG("Config: ({}) {} = {}", origin, key, log_value); + }); } // Guess compiler after logging the config value in order to be able to - // display "compiler_type = auto" before overwriting the value with the guess. + // display "compiler_type = auto" before overwriting the value with the + // guess. if (ctx.config.compiler_type() == CompilerType::auto_guess) { ctx.config.set_compiler_type(guess_compiler(ctx.orig_args[0])); } @@ -2371,8 +1999,7 @@ if (ctx.config.disable()) { LOG_RAW("ccache is disabled"); - // Statistic::cache_miss is a dummy to trigger stats_flush. - throw Failure(Statistic::cache_miss); + return nonstd::make_unexpected(Statistic::none); } LOG("Command line: {}", Util::format_argv_for_logging(argv)); @@ -2389,10 +2016,10 @@ MTR_END("main", "process_args"); if (processed.error) { - throw Failure(*processed.error); + return nonstd::make_unexpected(*processed.error); } - set_up_uncached_err(); + TRY(set_up_uncached_err()); if (ctx.config.depend_mode() && (!ctx.args_info.generating_dependencies @@ -2402,6 +2029,17 @@ ctx.config.set_depend_mode(false); } + if (ctx.storage.has_secondary_storage()) { + if (ctx.config.file_clone()) { + LOG_RAW("Disabling file clone mode since secondary storage is enabled"); + ctx.config.set_file_clone(false); + } + if (ctx.config.hard_link()) { + LOG_RAW("Disabling hard link mode since secondary storage is enabled"); + ctx.config.set_hard_link(false); + } + } + LOG("Source file: {}", ctx.args_info.input_file); if (ctx.args_info.generating_dependencies) { LOG("Dependency file: {}", ctx.args_info.output_dep); @@ -2440,10 +2078,11 @@ Hash common_hash; init_hash_debug(ctx, common_hash, 'c', "COMMON", debug_text_file); - MTR_BEGIN("hash", "common_hash"); - hash_common_info( - ctx, processed.preprocessor_args, common_hash, ctx.args_info); - MTR_END("hash", "common_hash"); + { + MTR_SCOPE("hash", "common_hash"); + TRY(hash_common_info( + ctx, processed.preprocessor_args, common_hash, ctx.args_info)); + } // Try to find the hash using the manifest. Hash direct_hash = common_hash; @@ -2453,38 +2092,45 @@ args_to_hash.push_back(processed.extra_args_to_hash); bool put_result_in_manifest = false; - optional result_name; - optional result_name_from_manifest; + optional result_key; + optional result_key_from_manifest; + optional manifest_key; + if (ctx.config.direct_mode()) { LOG_RAW("Trying direct lookup"); - MTR_BEGIN("hash", "direct_hash"); Args dummy_args; - result_name = - calculate_result_name(ctx, args_to_hash, dummy_args, direct_hash, true); + MTR_BEGIN("hash", "direct_hash"); + const auto result_and_manifest_key = calculate_result_and_manifest_key( + ctx, args_to_hash, dummy_args, direct_hash, true); MTR_END("hash", "direct_hash"); - if (result_name) { - ctx.set_result_name(*result_name); - + if (!result_and_manifest_key) { + return nonstd::make_unexpected(result_and_manifest_key.error()); + } + std::tie(result_key, manifest_key) = *result_and_manifest_key; + if (result_key) { // If we can return from cache at this point then do so. - auto result = from_cache(ctx, FromCacheCallMode::direct); - if (result) { - return *result; + const bool found = + from_cache(ctx, FromCacheCallMode::direct, *result_key); + if (found) { + return Statistic::direct_cache_hit; } // Wasn't able to return from cache at this point. However, the result // was already found in manifest, so don't re-add it later. put_result_in_manifest = false; - result_name_from_manifest = result_name; + result_key_from_manifest = result_key; } else { // Add result to manifest later. put_result_in_manifest = true; } + + ctx.storage.primary.increment_statistic(Statistic::direct_cache_miss); } if (ctx.config.read_only_direct()) { LOG_RAW("Read-only direct mode; running real compiler"); - throw Failure(Statistic::cache_miss); + return nonstd::make_unexpected(Statistic::cache_miss); } if (!ctx.config.depend_mode()) { @@ -2494,21 +2140,19 @@ init_hash_debug(ctx, cpp_hash, 'p', "PREPROCESSOR MODE", debug_text_file); MTR_BEGIN("hash", "cpp_hash"); - result_name = calculate_result_name( + const auto result_and_manifest_key = calculate_result_and_manifest_key( ctx, args_to_hash, processed.preprocessor_args, cpp_hash, false); MTR_END("hash", "cpp_hash"); + if (!result_and_manifest_key) { + return nonstd::make_unexpected(result_and_manifest_key.error()); + } + result_key = result_and_manifest_key->first; - // calculate_result_name does not return nullopt if the last (direct_mode) - // argument is false. - ASSERT(result_name); - ctx.set_result_name(*result_name); - - if (result_name_from_manifest && result_name_from_manifest != result_name) { - // manifest_path is guaranteed to be set when calculate_result_name - // returns a non-nullopt result in direct mode, i.e. when - // result_name_from_manifest is set. - ASSERT(ctx.manifest_path()); + // calculate_result_and_manifest_key always returns a non-nullopt result_key + // if the last argument (direct_mode) is false. + ASSERT(result_key); + if (result_key_from_manifest && result_key_from_manifest != result_key) { // The hash from manifest differs from the hash of the preprocessor // output. This could be because: // @@ -2524,24 +2168,26 @@ LOG_RAW("Hash from manifest doesn't match preprocessor output"); LOG_RAW("Likely reason: different CCACHE_BASEDIRs used"); LOG_RAW("Removing manifest as a safety measure"); - Util::unlink_safe(*ctx.manifest_path()); + ctx.storage.remove(*result_key, core::CacheEntryType::result); put_result_in_manifest = true; } // If we can return from cache at this point then do. - auto result = from_cache(ctx, FromCacheCallMode::cpp); - if (result) { - if (put_result_in_manifest) { - update_manifest_file(ctx); + const auto found = from_cache(ctx, FromCacheCallMode::cpp, *result_key); + if (found) { + if (ctx.config.direct_mode() && manifest_key && put_result_in_manifest) { + update_manifest_file(ctx, *manifest_key, *result_key); } - return *result; + return Statistic::preprocessed_cache_hit; } + + ctx.storage.primary.increment_statistic(Statistic::preprocessed_cache_miss); } if (ctx.config.read_only()) { LOG_RAW("Read-only mode; running real compiler"); - throw Failure(Statistic::cache_miss); + return nonstd::make_unexpected(Statistic::cache_miss); } add_prefix(ctx, processed.compiler_args, ctx.config.prefix_command()); @@ -2551,261 +2197,25 @@ // Run real compiler, sending output to cache. MTR_BEGIN("cache", "to_cache"); - to_cache(ctx, - processed.compiler_args, - ctx.args_info.depend_extra_args, - depend_mode_hash); - update_manifest_file(ctx); + const auto digest = to_cache(ctx, + processed.compiler_args, + result_key, + ctx.args_info.depend_extra_args, + depend_mode_hash); MTR_END("cache", "to_cache"); - - return Statistic::cache_miss; -} - -// The main program when not doing a compile. -static int -handle_main_options(int argc, const char* const* argv) -{ - enum longopts { - CHECKSUM_FILE, - CONFIG_PATH, - DUMP_MANIFEST, - DUMP_RESULT, - EVICT_OLDER_THAN, - EXTRACT_RESULT, - HASH_FILE, - PRINT_STATS, - }; - static const struct option options[] = { - {"checksum-file", required_argument, nullptr, CHECKSUM_FILE}, - {"cleanup", no_argument, nullptr, 'c'}, - {"clear", no_argument, nullptr, 'C'}, - {"config-path", required_argument, nullptr, CONFIG_PATH}, - {"directory", required_argument, nullptr, 'd'}, - {"dump-manifest", required_argument, nullptr, DUMP_MANIFEST}, - {"dump-result", required_argument, nullptr, DUMP_RESULT}, - {"evict-older-than", required_argument, nullptr, EVICT_OLDER_THAN}, - {"extract-result", required_argument, nullptr, EXTRACT_RESULT}, - {"get-config", required_argument, nullptr, 'k'}, - {"hash-file", required_argument, nullptr, HASH_FILE}, - {"help", no_argument, nullptr, 'h'}, - {"max-files", required_argument, nullptr, 'F'}, - {"max-size", required_argument, nullptr, 'M'}, - {"print-stats", no_argument, nullptr, PRINT_STATS}, - {"recompress", required_argument, nullptr, 'X'}, - {"set-config", required_argument, nullptr, 'o'}, - {"show-compression", no_argument, nullptr, 'x'}, - {"show-config", no_argument, nullptr, 'p'}, - {"show-stats", no_argument, nullptr, 's'}, - {"version", no_argument, nullptr, 'V'}, - {"zero-stats", no_argument, nullptr, 'z'}, - {nullptr, 0, nullptr, 0}}; - - Context ctx; - initialize(ctx, argc, argv); - - int c; - while ((c = getopt_long(argc, - const_cast(argv), - "cCd:k:hF:M:po:sVxX:z", - options, - nullptr)) - != -1) { - std::string arg = optarg ? optarg : std::string(); - - switch (c) { - case CHECKSUM_FILE: { - Checksum checksum; - Fd fd(arg == "-" ? STDIN_FILENO : open(arg.c_str(), O_RDONLY)); - Util::read_fd(*fd, [&checksum](const void* data, size_t size) { - checksum.update(data, size); - }); - PRINT(stdout, "{:016x}\n", checksum.digest()); - break; - } - - case CONFIG_PATH: - Util::setenv("CCACHE_CONFIGPATH", arg); - break; - - case DUMP_MANIFEST: - return Manifest::dump(arg, stdout) ? 0 : 1; - - case DUMP_RESULT: { - ResultDumper result_dumper(stdout); - Result::Reader result_reader(arg); - auto error = result_reader.read(result_dumper); - if (error) { - PRINT(stderr, "Error: {}\n", *error); - } - return error ? EXIT_FAILURE : EXIT_SUCCESS; - } - - case EVICT_OLDER_THAN: { - auto seconds = Util::parse_duration(arg); - ProgressBar progress_bar("Evicting..."); - clean_old( - ctx, [&](double progress) { progress_bar.update(progress); }, seconds); - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n"); - } - break; - } - - case EXTRACT_RESULT: { - ResultExtractor result_extractor("."); - Result::Reader result_reader(arg); - auto error = result_reader.read(result_extractor); - if (error) { - PRINT(stderr, "Error: {}\n", *error); - } - return error ? EXIT_FAILURE : EXIT_SUCCESS; - } - - case HASH_FILE: { - Hash hash; - if (arg == "-") { - hash.hash_fd(STDIN_FILENO); - } else { - hash.hash_file(arg); - } - PRINT(stdout, "{}\n", hash.digest().to_string()); - break; - } - - case PRINT_STATS: - PRINT_RAW(stdout, Statistics::format_machine_readable(ctx.config)); - break; - - case 'c': // --cleanup - { - ProgressBar progress_bar("Cleaning..."); - clean_up_all(ctx.config, - [&](double progress) { progress_bar.update(progress); }); - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n"); - } - break; - } - - case 'C': // --clear - { - ProgressBar progress_bar("Clearing..."); - wipe_all(ctx, [&](double progress) { progress_bar.update(progress); }); - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n"); - } - break; - } - - case 'd': // --directory - Util::setenv("CCACHE_DIR", arg); - break; - - case 'h': // --help - PRINT(stdout, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); - exit(EXIT_SUCCESS); - - case 'k': // --get-config - PRINT(stdout, "{}\n", ctx.config.get_string_value(arg)); - break; - - case 'F': { // --max-files - auto files = Util::parse_unsigned(arg); - Config::set_value_in_file( - ctx.config.primary_config_path(), "max_files", arg); - if (files == 0) { - PRINT_RAW(stdout, "Unset cache file limit\n"); - } else { - PRINT(stdout, "Set cache file limit to {}\n", files); - } - break; - } - - case 'M': { // --max-size - uint64_t size = Util::parse_size(arg); - Config::set_value_in_file( - ctx.config.primary_config_path(), "max_size", arg); - if (size == 0) { - PRINT_RAW(stdout, "Unset cache size limit\n"); - } else { - PRINT(stdout, - "Set cache size limit to {}\n", - Util::format_human_readable_size(size)); - } - break; - } - - case 'o': { // --set-config - // Start searching for equal sign at position 1 to improve error message - // for the -o=K=V case (key "=K" and value "V"). - size_t eq_pos = arg.find('=', 1); - if (eq_pos == std::string::npos) { - throw Error("missing equal sign in \"{}\"", arg); - } - std::string key = arg.substr(0, eq_pos); - std::string value = arg.substr(eq_pos + 1); - Config::set_value_in_file(ctx.config.primary_config_path(), key, value); - break; - } - - case 'p': // --show-config - ctx.config.visit_items(configuration_printer); - break; - - case 's': // --show-stats - PRINT_RAW(stdout, Statistics::format_human_readable(ctx.config)); - break; - - case 'V': // --version - PRINT(VERSION_TEXT, CCACHE_NAME, CCACHE_VERSION); - exit(EXIT_SUCCESS); - - case 'x': // --show-compression - { - ProgressBar progress_bar("Scanning..."); - compress_stats(ctx.config, - [&](double progress) { progress_bar.update(progress); }); - break; - } - - case 'X': // --recompress - { - optional wanted_level; - if (arg == "uncompressed") { - wanted_level = nullopt; - } else { - wanted_level = - Util::parse_signed(arg, INT8_MIN, INT8_MAX, "compression level"); - } - - ProgressBar progress_bar("Recompressing..."); - compress_recompress(ctx, wanted_level, [&](double progress) { - progress_bar.update(progress); - }); - break; - } - - case 'z': // --zero-stats - Statistics::zero_all_counters(ctx.config); - PRINT_RAW(stdout, "Statistics zeroed\n"); - break; - - default: - PRINT(stderr, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); - exit(EXIT_FAILURE); - } - - // Some of the above switches might have changed config settings, so run the - // setup again. - ctx.config = Config(); - set_up_config(ctx.config); + if (!digest) { + return nonstd::make_unexpected(digest.error()); + } + result_key = *digest; + if (ctx.config.direct_mode()) { + ASSERT(manifest_key); + MTR_SCOPE("cache", "update_manifest"); + update_manifest_file(ctx, *manifest_key, *result_key); } - return 0; + return ctx.config.recache() ? Statistic::recache : Statistic::cache_miss; } -int ccache_main(int argc, const char* const* argv); - int ccache_main(int argc, const char* const* argv) { @@ -2814,18 +2224,18 @@ std::string program_name(Util::base_name(argv[0])); if (Util::same_program_name(program_name, CCACHE_NAME)) { if (argc < 2) { - PRINT(stderr, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); + PRINT_RAW(stderr, core::get_usage_text()); exit(EXIT_FAILURE); } - // If the first argument isn't an option, then assume we are being passed - // a compiler name and options. + // If the first argument isn't an option, then assume we are being + // passed a compiler name and options. if (argv[1][0] == '-') { - return handle_main_options(argc, argv); + return core::process_main_options(argc, argv); } } return cache_compilation(argc, argv); - } catch (const ErrorBase& e) { + } catch (const core::ErrorBase& e) { PRINT(stderr, "ccache: error: {}\n", e.what()); return EXIT_FAILURE; } diff -Nru ccache-4.2.1/src/ccache.hpp ccache-4.5.1/src/ccache.hpp --- ccache-4.2.1/src/ccache.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ccache.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,5 +1,5 @@ // Copyright (C) 2002-2007 Andrew Tridgell -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors +// Copyright (C) 2009-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,8 +19,6 @@ #pragma once -#include "system.hpp" - #include "Config.hpp" #include "third_party/nonstd/string_view.hpp" @@ -30,6 +28,7 @@ class Context; +extern const char CCACHE_NAME[]; extern const char CCACHE_VERSION[]; using FindExecutableFunction = @@ -37,6 +36,8 @@ const std::string& name, const std::string& exclude_name)>; +int ccache_main(int argc, const char* const* argv); + // Tested by unit tests. void find_compiler(Context& ctx, const FindExecutableFunction& find_executable_function); diff -Nru ccache-4.2.1/src/Checksum.hpp ccache-4.5.1/src/Checksum.hpp --- ccache-4.2.1/src/Checksum.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Checksum.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,69 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#ifdef USE_XXH_DISPATCH -# include "third_party/xxh_x86dispatch.h" -#else -# include "third_party/xxhash.h" -#endif - -class Checksum -{ -public: - Checksum(); - ~Checksum(); - - void reset(); - void update(const void* data, size_t length); - uint64_t digest() const; - -private: - XXH3_state_t* m_state; -}; - -inline Checksum::Checksum() : m_state(XXH3_createState()) -{ - reset(); -} - -inline Checksum::~Checksum() -{ - XXH3_freeState(m_state); -} - -inline void -Checksum::reset() -{ - XXH3_64bits_reset(m_state); -} - -inline void -Checksum::update(const void* data, size_t length) -{ - XXH3_64bits_update(m_state, data, length); -} - -inline uint64_t -Checksum::digest() const -{ - return XXH3_64bits_digest(m_state); -} diff -Nru ccache-4.2.1/src/.clang-tidy ccache-4.5.1/src/.clang-tidy --- ccache-4.2.1/src/.clang-tidy 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/.clang-tidy 2021-11-17 19:31:58.000000000 +0000 @@ -69,11 +69,11 @@ # If you hit a limit, please consider changing the code instead of the limit. - key: readability-function-size.LineThreshold - value: 700 + value: 999999 - key: readability-function-size.StatementThreshold value: 999999 - key: readability-function-size.BranchThreshold - value: 170 + value: 999999 - key: readability-function-size.ParameterThreshold value: 6 - key: readability-function-size.NestingThreshold diff -Nru ccache-4.2.1/src/cleanup.cpp ccache-4.5.1/src/cleanup.cpp --- ccache-4.2.1/src/cleanup.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/cleanup.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,231 +0,0 @@ -// Copyright (C) 2002-2006 Andrew Tridgell -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "cleanup.hpp" - -#include "CacheFile.hpp" -#include "Config.hpp" -#include "Context.hpp" -#include "Logging.hpp" -#include "Statistics.hpp" -#include "Util.hpp" - -#ifdef INODE_CACHE_SUPPORTED -# include "InodeCache.hpp" -#endif - -#include - -static void -delete_file(const std::string& path, - uint64_t size, - uint64_t* cache_size, - uint64_t* files_in_cache) -{ - bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure); - if (!deleted && errno != ENOENT && errno != ESTALE) { - LOG("Failed to unlink {} ({})", path, strerror(errno)); - } else if (cache_size && files_in_cache) { - // The counters are intentionally subtracted even if there was no file to - // delete since the final cache size calculation will be incorrect if they - // aren't. (This can happen when there are several parallel ongoing - // cleanups of the same directory.) - *cache_size -= size; - --*files_in_cache; - } -} - -static void -update_counters(const std::string& dir, - uint64_t files_in_cache, - uint64_t cache_size, - bool cleanup_performed) -{ - const std::string stats_file = dir + "/stats"; - Statistics::update(stats_file, [=](Counters& cs) { - if (cleanup_performed) { - cs.increment(Statistic::cleanups_performed); - } - cs.set(Statistic::files_in_cache, files_in_cache); - cs.set(Statistic::cache_size_kibibyte, cache_size / 1024); - }); -} - -void -clean_old(const Context& ctx, - const Util::ProgressReceiver& progress_receiver, - uint64_t max_age) -{ - Util::for_each_level_1_subdir( - ctx.config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - clean_up_dir(subdir, 0, 0, max_age, sub_progress_receiver); - }, - progress_receiver); -} - -// Clean up one cache subdirectory. -void -clean_up_dir(const std::string& subdir, - uint64_t max_size, - uint64_t max_files, - uint64_t max_age, - const Util::ProgressReceiver& progress_receiver) -{ - LOG("Cleaning up cache directory {}", subdir); - - std::vector files = Util::get_level_1_files( - subdir, [&](double progress) { progress_receiver(progress / 3); }); - - uint64_t cache_size = 0; - uint64_t files_in_cache = 0; - time_t current_time = time(nullptr); - - for (size_t i = 0; i < files.size(); - ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) { - const auto& file = files[i]; - - if (!file.lstat().is_regular()) { - // Not a file or missing file. - continue; - } - - // Delete any tmp files older than 1 hour right away. - if (file.lstat().mtime() + 3600 < current_time - && Util::base_name(file.path()).find(".tmp.") != std::string::npos) { - Util::unlink_tmp(file.path()); - continue; - } - - cache_size += file.lstat().size_on_disk(); - files_in_cache += 1; - } - - // Sort according to modification time, oldest first. - std::sort( - files.begin(), files.end(), [](const CacheFile& f1, const CacheFile& f2) { - return f1.lstat().mtime() < f2.lstat().mtime(); - }); - - LOG("Before cleanup: {:.0f} KiB, {:.0f} files", - static_cast(cache_size) / 1024, - static_cast(files_in_cache)); - - bool cleaned = false; - for (size_t i = 0; i < files.size(); - ++i, progress_receiver(2.0 / 3 + 1.0 * i / files.size() / 3)) { - const auto& file = files[i]; - - if (!file.lstat() || file.lstat().is_directory()) { - continue; - } - - if ((max_size == 0 || cache_size <= max_size) - && (max_files == 0 || files_in_cache <= max_files) - && (max_age == 0 - || file.lstat().mtime() - > (current_time - static_cast(max_age)))) { - break; - } - - if (Util::ends_with(file.path(), ".stderr")) { - // In order to be nice to legacy ccache versions, make sure that the .o - // file is deleted before .stderr, because if the ccache process gets - // killed after deleting the .stderr but before deleting the .o, the - // cached result will be inconsistent. (.stderr is the only file that is - // optional for legacy ccache versions; any other file missing from the - // cache will be detected.) - std::string o_file = file.path().substr(0, file.path().size() - 6) + "o"; - - // Don't subtract this extra deletion from the cache size; that - // bookkeeping will be done when the loop reaches the .o file. If the - // loop doesn't reach the .o file since the target limits have been - // reached, the bookkeeping won't happen, but that small counter - // discrepancy won't do much harm and it will correct itself in the next - // cleanup. - delete_file(o_file, 0, nullptr, nullptr); - } - - delete_file( - file.path(), file.lstat().size_on_disk(), &cache_size, &files_in_cache); - cleaned = true; - } - - LOG("After cleanup: {:.0f} KiB, {:.0f} files", - static_cast(cache_size) / 1024, - static_cast(files_in_cache)); - - if (cleaned) { - LOG("Cleaned up cache directory {}", subdir); - } - - update_counters(subdir, files_in_cache, cache_size, cleaned); -} - -// Clean up all cache subdirectories. -void -clean_up_all(const Config& config, - const Util::ProgressReceiver& progress_receiver) -{ - Util::for_each_level_1_subdir( - config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - clean_up_dir(subdir, - config.max_size() / 16, - config.max_files() / 16, - 0, - sub_progress_receiver); - }, - progress_receiver); -} - -// Wipe one cache subdirectory. -static void -wipe_dir(const std::string& subdir, - const Util::ProgressReceiver& progress_receiver) -{ - LOG("Clearing out cache directory {}", subdir); - - const std::vector files = Util::get_level_1_files( - subdir, [&](double progress) { progress_receiver(progress / 2); }); - - for (size_t i = 0; i < files.size(); ++i) { - Util::unlink_safe(files[i].path()); - progress_receiver(0.5 + 0.5 * i / files.size()); - } - - const bool cleared = !files.empty(); - if (cleared) { - LOG("Cleared out cache directory {}", subdir); - } - update_counters(subdir, 0, 0, cleared); -} - -// Wipe all cached files in all subdirectories. -void -wipe_all(const Context& ctx, const Util::ProgressReceiver& progress_receiver) -{ - Util::for_each_level_1_subdir( - ctx.config.cache_dir(), wipe_dir, progress_receiver); -#ifdef INODE_CACHE_SUPPORTED - ctx.inode_cache.drop(); -#endif -} diff -Nru ccache-4.2.1/src/cleanup.hpp ccache-4.5.1/src/cleanup.hpp --- ccache-4.2.1/src/cleanup.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/cleanup.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,44 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Util.hpp" - -#include - -class Config; -class Context; - -void clean_old(const Context& ctx, - const Util::ProgressReceiver& progress_receiver, - uint64_t max_age); - -void clean_up_dir(const std::string& subdir, - uint64_t max_size, - uint64_t max_files, - uint64_t max_age, - const Util::ProgressReceiver& progress_receiver); - -void clean_up_all(const Config& config, - const Util::ProgressReceiver& progress_receiver); - -void wipe_all(const Context& ctx, - const Util::ProgressReceiver& progress_receiver); diff -Nru ccache-4.2.1/src/CMakeLists.txt ccache-4.5.1/src/CMakeLists.txt --- ccache-4.2.1/src/CMakeLists.txt 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -2,23 +2,14 @@ source_files Args.cpp AtomicFile.cpp - CacheEntryReader.cpp - CacheEntryWriter.cpp - CacheFile.cpp - Compression.cpp - Compressor.cpp Config.cpp Context.cpp - Counters.cpp - Decompressor.cpp Depfile.cpp + Fd.cpp Hash.cpp Lockfile.cpp Logging.cpp Manifest.cpp - MiniTrace.cpp - NullCompressor.cpp - NullDecompressor.cpp ProgressBar.cpp Result.cpp ResultDumper.cpp @@ -26,57 +17,69 @@ ResultRetriever.cpp SignalHandler.cpp Stat.cpp - Statistics.cpp TemporaryFile.cpp ThreadPool.cpp Util.cpp - ZstdCompressor.cpp - ZstdDecompressor.cpp argprocessing.cpp assertions.cpp ccache.cpp - cleanup.cpp compopt.cpp - compress.cpp execute.cpp hashutil.cpp language.cpp - version.cpp) + version.cpp +) if(INODE_CACHE_SUPPORTED) list(APPEND source_files InodeCache.cpp) endif() +if(MTR_ENABLED) + list(APPEND source_files MiniTrace.cpp) +endif() + if(WIN32) list(APPEND source_files Win32Util.cpp) endif() -add_library(ccache_lib STATIC ${source_files}) +add_library(ccache_framework STATIC ${source_files}) target_compile_definitions( - ccache_lib PUBLIC -Dnssv_CONFIG_SELECT_STRING_VIEW=nssv_STRING_VIEW_NONSTD + ccache_framework PUBLIC -Dnssv_CONFIG_SELECT_STRING_VIEW=nssv_STRING_VIEW_NONSTD ) if(WIN32) - target_link_libraries(ccache_lib PRIVATE ws2_32 "psapi") + target_link_libraries(ccache_framework PRIVATE "psapi") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") if(STATIC_LINK) - target_link_libraries(ccache_lib PRIVATE -static-libgcc -static-libstdc++ -static winpthread -dynamic) + target_link_libraries(ccache_framework PRIVATE -static-libgcc -static-libstdc++ -static winpthread -dynamic) else() - target_link_libraries(ccache_lib PRIVATE winpthread) + target_link_libraries(ccache_framework PRIVATE winpthread) endif() elseif(STATIC_LINK AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - target_link_libraries(ccache_lib PRIVATE -static c++ -dynamic) + target_link_libraries(ccache_framework PRIVATE -static c++ -dynamic) endif() endif() set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) target_link_libraries( - ccache_lib - PRIVATE standard_settings standard_warnings ZSTD::ZSTD - Threads::Threads third_party_lib) + ccache_framework + PRIVATE standard_settings standard_warnings ZSTD::ZSTD Threads::Threads third_party +) + +target_include_directories(ccache_framework PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -target_include_directories(ccache_lib PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +if(REDIS_STORAGE_BACKEND) + target_compile_definitions(ccache_framework PRIVATE -DHAVE_REDIS_STORAGE_BACKEND) + target_link_libraries( + ccache_framework + PUBLIC standard_settings standard_warnings HIREDIS::HIREDIS third_party + ) +endif() +add_subdirectory(compression) +add_subdirectory(core) +add_subdirectory(storage) add_subdirectory(third_party) +add_subdirectory(util) diff -Nru ccache-4.2.1/src/compopt.cpp ccache-4.5.1/src/compopt.cpp --- ccache-4.2.1/src/compopt.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/compopt.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -45,6 +45,8 @@ // The option only affects compilation; not passed to the preprocessor. #define AFFECTS_COMP (1 << 6) +#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) + struct CompOpt { const char* name; @@ -52,16 +54,19 @@ }; const CompOpt compopts[] = { - {"--Werror", TAKES_ARG}, // nvcc - {"--analyze", TOO_HARD}, // Clang - {"--compiler-bindir", AFFECTS_CPP | TAKES_ARG}, // nvcc - {"--libdevice-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc - {"--output-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc + {"--Werror", TAKES_ARG}, // nvcc + {"--analyze", TOO_HARD}, // Clang + {"--compiler-bindir", AFFECTS_CPP | TAKES_ARG}, // nvcc + {"--config", TAKES_ARG}, // Clang + {"--gcc-toolchain=", TAKES_CONCAT_ARG | TAKES_PATH}, // Clang + {"--libdevice-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc + {"--output-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc {"--param", TAKES_ARG}, {"--save-temps", TOO_HARD}, {"--save-temps=cwd", TOO_HARD}, {"--save-temps=obj", TOO_HARD}, {"--serialize-diagnostics", TAKES_ARG | TAKES_PATH}, + {"--specs", TAKES_ARG}, {"-A", TAKES_ARG}, {"-B", TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-D", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG}, @@ -76,7 +81,6 @@ {"-MM", TOO_HARD}, {"-MQ", TAKES_ARG}, {"-MT", TAKES_ARG}, - {"-P", TOO_HARD}, {"-U", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG}, {"-V", TAKES_ARG}, {"-Wa,", TAKES_CONCAT_ARG | AFFECTS_COMP}, @@ -102,6 +106,7 @@ {"-frepo", TOO_HARD}, {"-ftime-trace", TOO_HARD}, // Clang {"-fworking-directory", AFFECTS_CPP}, + {"-gcc-toolchain", TAKES_ARG | TAKES_PATH}, // Clang {"-gen-cdb-fragment-path", TAKES_ARG | TOO_HARD}, // Clang {"-gtoggle", TOO_HARD}, {"-idirafter", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, @@ -116,6 +121,7 @@ {"-iquote", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-isysroot", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-isystem", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, + {"-ivfsoverlay", TAKES_ARG}, {"-iwithprefix", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-iwithprefixbefore", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, @@ -132,6 +138,7 @@ {"-save-temps", TOO_HARD}, {"-save-temps=cwd", TOO_HARD}, {"-save-temps=obj", TOO_HARD}, + {"-specs", TAKES_ARG}, {"-stdlib=", AFFECTS_CPP | TAKES_CONCAT_ARG}, {"-trigraphs", AFFECTS_CPP}, {"-u", TAKES_ARG | TAKES_CONCAT_ARG}, diff -Nru ccache-4.2.1/src/compopt.hpp ccache-4.5.1/src/compopt.hpp --- ccache-4.2.1/src/compopt.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/compopt.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors +// Copyright (C) 2010-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include bool compopt_short(bool (*fn)(const std::string& option), diff -Nru ccache-4.2.1/src/compress.cpp ccache-4.5.1/src/compress.cpp --- ccache-4.2.1/src/compress.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/compress.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,372 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "compress.hpp" - -#include "AtomicFile.hpp" -#include "CacheEntryReader.hpp" -#include "CacheEntryWriter.hpp" -#include "Context.hpp" -#include "File.hpp" -#include "Logging.hpp" -#include "Manifest.hpp" -#include "Result.hpp" -#include "Statistics.hpp" -#include "StdMakeUnique.hpp" -#include "ThreadPool.hpp" -#include "ZstdCompressor.hpp" -#include "assertions.hpp" -#include "fmtmacros.hpp" - -#include "third_party/fmt/core.h" - -#include -#include - -using nonstd::optional; - -namespace { - -class RecompressionStatistics -{ -public: - void update(uint64_t content_size, - uint64_t old_size, - uint64_t new_size, - uint64_t incompressible_size); - uint64_t content_size() const; - uint64_t old_size() const; - uint64_t new_size() const; - uint64_t incompressible_size() const; - -private: - mutable std::mutex m_mutex; - uint64_t m_content_size = 0; - uint64_t m_old_size = 0; - uint64_t m_new_size = 0; - uint64_t m_incompressible_size = 0; -}; - -void -RecompressionStatistics::update(uint64_t content_size, - uint64_t old_size, - uint64_t new_size, - uint64_t incompressible_size) -{ - std::unique_lock lock(m_mutex); - m_incompressible_size += incompressible_size; - m_content_size += content_size; - m_old_size += old_size; - m_new_size += new_size; -} - -uint64_t -RecompressionStatistics::content_size() const -{ - std::unique_lock lock(m_mutex); - return m_content_size; -} - -uint64_t -RecompressionStatistics::old_size() const -{ - std::unique_lock lock(m_mutex); - return m_old_size; -} - -uint64_t -RecompressionStatistics::new_size() const -{ - std::unique_lock lock(m_mutex); - return m_new_size; -} - -uint64_t -RecompressionStatistics::incompressible_size() const -{ - std::unique_lock lock(m_mutex); - return m_incompressible_size; -} - -File -open_file(const std::string& path, const char* mode) -{ - File f(path, mode); - if (!f) { - throw Error("failed to open {} for reading: {}", path, strerror(errno)); - } - return f; -} - -std::unique_ptr -create_reader(const CacheFile& cache_file, FILE* stream) -{ - if (cache_file.type() == CacheFile::Type::unknown) { - throw Error("unknown file type for {}", cache_file.path()); - } - - switch (cache_file.type()) { - case CacheFile::Type::result: - return std::make_unique( - stream, Result::k_magic, Result::k_version); - - case CacheFile::Type::manifest: - return std::make_unique( - stream, Manifest::k_magic, Manifest::k_version); - - case CacheFile::Type::unknown: - ASSERT(false); // Handled at function entry. - } - - ASSERT(false); -} - -std::unique_ptr -create_writer(FILE* stream, - const CacheEntryReader& reader, - Compression::Type compression_type, - int8_t compression_level) -{ - return std::make_unique(stream, - reader.magic(), - reader.version(), - compression_type, - compression_level, - reader.payload_size()); -} - -void -recompress_file(RecompressionStatistics& statistics, - const std::string& stats_file, - const CacheFile& cache_file, - optional level) -{ - auto file = open_file(cache_file.path(), "rb"); - auto reader = create_reader(cache_file, file.get()); - - auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log); - uint64_t content_size = reader->content_size(); - int8_t wanted_level = - level ? (*level == 0 ? ZstdCompressor::default_compression_level : *level) - : 0; - - if (reader->compression_level() == wanted_level) { - statistics.update(content_size, old_stat.size(), old_stat.size(), 0); - return; - } - - LOG("Recompressing {} to {}", - cache_file.path(), - level ? FMT("level {}", wanted_level) : "uncompressed"); - AtomicFile atomic_new_file(cache_file.path(), AtomicFile::Mode::binary); - auto writer = - create_writer(atomic_new_file.stream(), - *reader, - level ? Compression::Type::zstd : Compression::Type::none, - wanted_level); - - char buffer[READ_BUFFER_SIZE]; - size_t bytes_left = reader->payload_size(); - while (bytes_left > 0) { - size_t bytes_to_read = std::min(bytes_left, sizeof(buffer)); - reader->read(buffer, bytes_to_read); - writer->write(buffer, bytes_to_read); - bytes_left -= bytes_to_read; - } - reader->finalize(); - writer->finalize(); - - file.close(); - - atomic_new_file.commit(); - auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log); - - Statistics::update(stats_file, [=](Counters& cs) { - cs.increment(Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(old_stat, new_stat)); - }); - - statistics.update(content_size, old_stat.size(), new_stat.size(), 0); - - LOG("Recompression of {} done", cache_file.path()); -} - -} // namespace - -void -compress_stats(const Config& config, - const Util::ProgressReceiver& progress_receiver) -{ - uint64_t on_disk_size = 0; - uint64_t compr_size = 0; - uint64_t content_size = 0; - uint64_t incompr_size = 0; - - Util::for_each_level_1_subdir( - config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - const std::vector files = Util::get_level_1_files( - subdir, [&](double progress) { sub_progress_receiver(progress / 2); }); - - for (size_t i = 0; i < files.size(); ++i) { - const auto& cache_file = files[i]; - on_disk_size += cache_file.lstat().size_on_disk(); - - try { - auto file = open_file(cache_file.path(), "rb"); - auto reader = create_reader(cache_file, file.get()); - compr_size += cache_file.lstat().size(); - content_size += reader->content_size(); - } catch (Error&) { - incompr_size += cache_file.lstat().size(); - } - - sub_progress_receiver(1.0 / 2 + 1.0 * i / files.size() / 2); - } - }, - progress_receiver); - - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n\n"); - } - - double ratio = - compr_size > 0 ? static_cast(content_size) / compr_size : 0.0; - double savings = ratio > 0.0 ? 100.0 - (100.0 / ratio) : 0.0; - - std::string on_disk_size_str = Util::format_human_readable_size(on_disk_size); - std::string cache_size_str = - Util::format_human_readable_size(compr_size + incompr_size); - std::string compr_size_str = Util::format_human_readable_size(compr_size); - std::string content_size_str = Util::format_human_readable_size(content_size); - std::string incompr_size_str = Util::format_human_readable_size(incompr_size); - - PRINT(stdout, - "Total data: {:>8s} ({} disk blocks)\n", - cache_size_str, - on_disk_size_str); - PRINT(stdout, - "Compressed data: {:>8s} ({:.1f}% of original size)\n", - compr_size_str, - 100.0 - savings); - PRINT(stdout, " - Original data: {:>8s}\n", content_size_str); - PRINT(stdout, - " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", - ratio, - savings); - PRINT(stdout, "Incompressible data: {:>8s}\n", incompr_size_str); -} - -void -compress_recompress(Context& ctx, - optional level, - const Util::ProgressReceiver& progress_receiver) -{ - const size_t threads = std::thread::hardware_concurrency(); - const size_t read_ahead = 2 * threads; - ThreadPool thread_pool(threads, read_ahead); - RecompressionStatistics statistics; - - Util::for_each_level_1_subdir( - ctx.config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - std::vector files = - Util::get_level_1_files(subdir, [&](double progress) { - sub_progress_receiver(0.1 * progress); - }); - - auto stats_file = subdir + "/stats"; - - for (size_t i = 0; i < files.size(); ++i) { - const auto& file = files[i]; - - if (file.type() != CacheFile::Type::unknown) { - thread_pool.enqueue([&statistics, stats_file, file, level] { - try { - recompress_file(statistics, stats_file, file, level); - } catch (Error&) { - // Ignore for now. - } - }); - } else { - statistics.update(0, 0, 0, file.lstat().size()); - } - - sub_progress_receiver(0.1 + 0.9 * i / files.size()); - } - - if (Util::ends_with(subdir, "f")) { - // Wait here instead of after Util::for_each_level_1_subdir to avoid - // updating the progress bar to 100% before all work is done. - thread_pool.shut_down(); - } - }, - progress_receiver); - - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n\n"); - } - - double old_ratio = - statistics.old_size() > 0 - ? static_cast(statistics.content_size()) / statistics.old_size() - : 0.0; - double old_savings = old_ratio > 0.0 ? 100.0 - (100.0 / old_ratio) : 0.0; - double new_ratio = - statistics.new_size() > 0 - ? static_cast(statistics.content_size()) / statistics.new_size() - : 0.0; - double new_savings = new_ratio > 0.0 ? 100.0 - (100.0 / new_ratio) : 0.0; - int64_t size_difference = static_cast(statistics.new_size()) - - static_cast(statistics.old_size()); - - std::string old_compr_size_str = - Util::format_human_readable_size(statistics.old_size()); - std::string new_compr_size_str = - Util::format_human_readable_size(statistics.new_size()); - std::string content_size_str = - Util::format_human_readable_size(statistics.content_size()); - std::string incompr_size_str = - Util::format_human_readable_size(statistics.incompressible_size()); - std::string size_difference_str = - FMT("{}{}", - size_difference < 0 ? "-" : (size_difference > 0 ? "+" : " "), - Util::format_human_readable_size( - size_difference < 0 ? -size_difference : size_difference)); - - PRINT(stdout, "Original data: {:>8s}\n", content_size_str); - PRINT(stdout, - "Old compressed data: {:>8s} ({:.1f}% of original size)\n", - old_compr_size_str, - 100.0 - old_savings); - PRINT(stdout, - " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", - old_ratio, - old_savings); - PRINT(stdout, - "New compressed data: {:>8s} ({:.1f}% of original size)\n", - new_compr_size_str, - 100.0 - new_savings); - PRINT(stdout, - " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", - new_ratio, - new_savings); - PRINT(stdout, "Size change: {:>9s}\n", size_difference_str); -} diff -Nru ccache-4.2.1/src/compress.hpp ccache-4.5.1/src/compress.hpp --- ccache-4.2.1/src/compress.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/compress.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,42 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Util.hpp" - -#include "third_party/nonstd/optional.hpp" - -class Config; -class Context; - -void compress_stats(const Config& config, - const Util::ProgressReceiver& progress_receiver); - -// Recompress the cache. -// -// Arguments: -// - ctx: The context. -// - level: Target compression level (positive or negative value for actual -// level, 0 for default level and nonstd::nullopt for no compression). -// - progress_receiver: Function that will be called for progress updates. -void compress_recompress(Context& ctx, - nonstd::optional level, - const Util::ProgressReceiver& progress_receiver); diff -Nru ccache-4.2.1/src/compression/CMakeLists.txt ccache-4.5.1/src/compression/CMakeLists.txt --- ccache-4.2.1/src/compression/CMakeLists.txt 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,12 @@ +set( + sources + ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Decompressor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/NullCompressor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/NullDecompressor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ZstdCompressor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ZstdDecompressor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/types.cpp +) + +target_sources(ccache_framework PRIVATE ${sources}) diff -Nru ccache-4.2.1/src/compression/Compressor.cpp ccache-4.5.1/src/compression/Compressor.cpp --- ccache-4.2.1/src/compression/Compressor.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/Compressor.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,47 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Compressor.hpp" + +#include "NullCompressor.hpp" +#include "ZstdCompressor.hpp" +#include "assertions.hpp" + +#include + +#include + +namespace compression { + +std::unique_ptr +Compressor::create_from_type(const Type type, + core::Writer& writer, + const int8_t compression_level) +{ + switch (type) { + case compression::Type::none: + return std::make_unique(writer); + + case compression::Type::zstd: + return std::make_unique(writer, compression_level); + } + + ASSERT(false); +} + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/Compressor.hpp ccache-4.5.1/src/compression/Compressor.hpp --- ccache-4.2.1/src/compression/Compressor.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/Compressor.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,46 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include +#include + +namespace core { + +class Writer; + +} + +namespace compression { + +class Compressor : public core::Writer +{ +public: + virtual ~Compressor() = default; + + static std::unique_ptr + create_from_type(Type type, core::Writer& writer, int8_t compression_level); + + virtual int8_t actual_compression_level() const = 0; +}; + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/Decompressor.cpp ccache-4.5.1/src/compression/Decompressor.cpp --- ccache-4.2.1/src/compression/Decompressor.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/Decompressor.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,41 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Decompressor.hpp" + +#include "NullDecompressor.hpp" +#include "ZstdDecompressor.hpp" +#include "assertions.hpp" + +namespace compression { + +std::unique_ptr +Decompressor::create_from_type(Type type, core::Reader& reader) +{ + switch (type) { + case compression::Type::none: + return std::make_unique(reader); + + case compression::Type::zstd: + return std::make_unique(reader); + } + + ASSERT(false); +} + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/Decompressor.hpp ccache-4.5.1/src/compression/Decompressor.hpp --- ccache-4.2.1/src/compression/Decompressor.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/Decompressor.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include + +namespace compression { + +class Decompressor : public core::Reader +{ +public: + virtual ~Decompressor() = default; + + // Create a decompressor for the specified type. + static std::unique_ptr create_from_type(Type type, + core::Reader& reader); + + // Finalize decompression. + // + // This method checks that the end state of the compressed stream is correct + // and throws Error if not. + virtual void finalize() = 0; +}; + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/NullCompressor.cpp ccache-4.5.1/src/compression/NullCompressor.cpp --- ccache-4.2.1/src/compression/NullCompressor.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/NullCompressor.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,47 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "NullCompressor.hpp" + +#include + +namespace compression { + +NullCompressor::NullCompressor(core::Writer& writer) : m_writer(writer) +{ +} + +int8_t +NullCompressor::actual_compression_level() const +{ + return 0; +} + +void +NullCompressor::write(const void* const data, const size_t count) +{ + m_writer.write(data, count); +} + +void +NullCompressor::finalize() +{ + m_writer.finalize(); +} + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/NullCompressor.hpp ccache-4.5.1/src/compression/NullCompressor.hpp --- ccache-4.2.1/src/compression/NullCompressor.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/NullCompressor.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,41 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Compressor.hpp" + +#include + +namespace compression { + +// A compressor of an uncompressed stream. +class NullCompressor : public Compressor, NonCopyable +{ +public: + explicit NullCompressor(core::Writer& writer); + + int8_t actual_compression_level() const override; + void write(const void* data, size_t count) override; + void finalize() override; + +private: + core::Writer& m_writer; +}; + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/NullDecompressor.cpp ccache-4.5.1/src/compression/NullDecompressor.cpp --- ccache-4.2.1/src/compression/NullDecompressor.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/NullDecompressor.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,50 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "NullDecompressor.hpp" + +#include + +namespace compression { + +NullDecompressor::NullDecompressor(core::Reader& reader) : m_reader(reader) +{ +} + +size_t +NullDecompressor::read(void* const data, const size_t count) +{ + return m_reader.read(data, count); +} + +void +NullDecompressor::finalize() +{ + bool eof; + try { + m_reader.read_int(); + eof = false; + } catch (core::Error&) { + eof = true; + } + if (!eof) { + throw core::Error("Garbage data at end of uncompressed stream"); + } +} + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/NullDecompressor.hpp ccache-4.5.1/src/compression/NullDecompressor.hpp --- ccache-4.2.1/src/compression/NullDecompressor.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/NullDecompressor.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,40 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Decompressor.hpp" + +#include + +namespace compression { + +// A decompressor of an uncompressed stream. +class NullDecompressor : public Decompressor, NonCopyable +{ +public: + explicit NullDecompressor(core::Reader& reader); + + size_t read(void* data, size_t count) override; + void finalize() override; + +private: + core::Reader& m_reader; +}; + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/types.cpp ccache-4.5.1/src/compression/types.cpp --- ccache-4.2.1/src/compression/types.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/types.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,68 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "types.hpp" + +#include +#include +#include +#include + +namespace compression { + +int8_t +level_from_config(const Config& config) +{ + return config.compression() ? config.compression_level() : 0; +} + +Type +type_from_config(const Config& config) +{ + return config.compression() ? Type::zstd : Type::none; +} + +Type +type_from_int(const uint8_t type) +{ + switch (type) { + case static_cast(Type::none): + return Type::none; + + case static_cast(Type::zstd): + return Type::zstd; + } + + throw core::Error("Unknown type: {}", type); +} + +std::string +type_to_string(const Type type) +{ + switch (type) { + case Type::none: + return "none"; + + case Type::zstd: + return "zstd"; + } + + ASSERT(false); +} + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/types.hpp ccache-4.5.1/src/compression/types.hpp --- ccache-4.2.1/src/compression/types.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/types.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,41 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +class Config; + +namespace compression { + +enum class Type : uint8_t { + none = 0, + zstd = 1, +}; + +int8_t level_from_config(const Config& config); + +Type type_from_config(const Config& config); + +Type type_from_int(uint8_t type); + +std::string type_to_string(Type type); + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/ZstdCompressor.cpp ccache-4.5.1/src/compression/ZstdCompressor.cpp --- ccache-4.2.1/src/compression/ZstdCompressor.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/ZstdCompressor.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,122 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ZstdCompressor.hpp" + +#include "Logging.hpp" +#include "assertions.hpp" + +#include + +#include + +#include + +namespace compression { + +ZstdCompressor::ZstdCompressor(core::Writer& writer, int8_t compression_level) + : m_writer(writer), + m_zstd_stream(ZSTD_createCStream()), + m_zstd_in(std::make_unique()), + m_zstd_out(std::make_unique()) +{ + if (compression_level == 0) { + compression_level = default_compression_level; + LOG("Using default compression level {}", compression_level); + } + + // libzstd 1.3.4 and newer support negative levels. However, the query + // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection + // based on version instead. + if (ZSTD_versionNumber() < 10304 && compression_level < 1) { + LOG( + "Using compression level 1 (minimum level supported by libzstd) instead" + " of {}", + compression_level); + compression_level = 1; + } + + m_compression_level = std::min(compression_level, ZSTD_maxCLevel()); + if (m_compression_level != compression_level) { + LOG("Using compression level {} (max libzstd level) instead of {}", + m_compression_level, + compression_level); + } + + size_t ret = ZSTD_initCStream(m_zstd_stream, m_compression_level); + if (ZSTD_isError(ret)) { + ZSTD_freeCStream(m_zstd_stream); + throw core::Error("error initializing zstd compression stream"); + } +} + +ZstdCompressor::~ZstdCompressor() +{ + ZSTD_freeCStream(m_zstd_stream); +} + +int8_t +ZstdCompressor::actual_compression_level() const +{ + return m_compression_level; +} + +void +ZstdCompressor::write(const void* const data, const size_t count) +{ + m_zstd_in->src = data; + m_zstd_in->size = count; + m_zstd_in->pos = 0; + + int flush = data ? 0 : 1; + + size_t ret; + while (m_zstd_in->pos < m_zstd_in->size) { + uint8_t buffer[CCACHE_READ_BUFFER_SIZE]; + m_zstd_out->dst = buffer; + m_zstd_out->size = sizeof(buffer); + m_zstd_out->pos = 0; + ret = ZSTD_compressStream(m_zstd_stream, m_zstd_out.get(), m_zstd_in.get()); + ASSERT(!(ZSTD_isError(ret))); + const size_t compressed_bytes = m_zstd_out->pos; + if (compressed_bytes > 0) { + m_writer.write(buffer, compressed_bytes); + } + } + ret = flush; + while (ret > 0) { + uint8_t buffer[CCACHE_READ_BUFFER_SIZE]; + m_zstd_out->dst = buffer; + m_zstd_out->size = sizeof(buffer); + m_zstd_out->pos = 0; + ret = ZSTD_endStream(m_zstd_stream, m_zstd_out.get()); + const size_t compressed_bytes = m_zstd_out->pos; + if (compressed_bytes > 0) { + m_writer.write(buffer, compressed_bytes); + } + } +} + +void +ZstdCompressor::finalize() +{ + write(nullptr, 0); + m_writer.finalize(); +} + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/ZstdCompressor.hpp ccache-4.5.1/src/compression/ZstdCompressor.hpp --- ccache-4.2.1/src/compression/ZstdCompressor.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/ZstdCompressor.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,56 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Compressor.hpp" + +#include + +#include +#include + +struct ZSTD_CCtx_s; +struct ZSTD_inBuffer_s; +struct ZSTD_outBuffer_s; + +namespace compression { + +// A compressor of a Zstandard stream. +class ZstdCompressor : public Compressor, NonCopyable +{ +public: + ZstdCompressor(core::Writer& writer, int8_t compression_level); + + ~ZstdCompressor() override; + + int8_t actual_compression_level() const override; + void write(const void* data, size_t count) override; + void finalize() override; + + constexpr static uint8_t default_compression_level = 1; + +private: + core::Writer& m_writer; + ZSTD_CCtx_s* m_zstd_stream; + std::unique_ptr m_zstd_in; + std::unique_ptr m_zstd_out; + int8_t m_compression_level; +}; + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/ZstdDecompressor.cpp ccache-4.5.1/src/compression/ZstdDecompressor.cpp --- ccache-4.2.1/src/compression/ZstdDecompressor.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/ZstdDecompressor.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,88 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ZstdDecompressor.hpp" + +#include "assertions.hpp" + +#include + +namespace compression { + +ZstdDecompressor::ZstdDecompressor(core::Reader& reader) + : m_reader(reader), + m_input_size(0), + m_input_consumed(0), + m_zstd_stream(ZSTD_createDStream()), + m_reached_stream_end(false) +{ + const size_t ret = ZSTD_initDStream(m_zstd_stream); + if (ZSTD_isError(ret)) { + ZSTD_freeDStream(m_zstd_stream); + throw core::Error("failed to initialize zstd decompression stream"); + } +} + +ZstdDecompressor::~ZstdDecompressor() +{ + ZSTD_freeDStream(m_zstd_stream); +} + +size_t +ZstdDecompressor::read(void* const data, const size_t count) +{ + size_t bytes_read = 0; + while (bytes_read < count) { + ASSERT(m_input_size >= m_input_consumed); + if (m_input_size == m_input_consumed) { + m_input_size = m_reader.read(m_input_buffer, sizeof(m_input_buffer)); + m_input_consumed = 0; + } + + m_zstd_in.src = (m_input_buffer + m_input_consumed); + m_zstd_in.size = m_input_size - m_input_consumed; + m_zstd_in.pos = 0; + + m_zstd_out.dst = static_cast(data) + bytes_read; + m_zstd_out.size = count - bytes_read; + m_zstd_out.pos = 0; + const size_t ret = + ZSTD_decompressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in); + if (ZSTD_isError(ret)) { + throw core::Error("Failed to read from zstd input stream"); + } + if (ret == 0) { + m_reached_stream_end = true; + break; + } + bytes_read += m_zstd_out.pos; + m_input_consumed += m_zstd_in.pos; + } + + return count; +} + +void +ZstdDecompressor::finalize() +{ + if (!m_reached_stream_end) { + throw core::Error("Garbage data at end of zstd input stream"); + } +} + +} // namespace compression diff -Nru ccache-4.2.1/src/compression/ZstdDecompressor.hpp ccache-4.5.1/src/compression/ZstdDecompressor.hpp --- ccache-4.2.1/src/compression/ZstdDecompressor.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/compression/ZstdDecompressor.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,51 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Decompressor.hpp" + +#include + +#include + +namespace compression { + +// A decompressor of a Zstandard stream. +class ZstdDecompressor : public Decompressor +{ +public: + explicit ZstdDecompressor(core::Reader& reader); + + ~ZstdDecompressor() override; + + size_t read(void* data, size_t count) override; + void finalize() override; + +private: + core::Reader& m_reader; + char m_input_buffer[CCACHE_READ_BUFFER_SIZE]; + size_t m_input_size; + size_t m_input_consumed; + ZSTD_DStream* m_zstd_stream; + ZSTD_inBuffer m_zstd_in; + ZSTD_outBuffer m_zstd_out; + bool m_reached_stream_end; +}; + +} // namespace compression diff -Nru ccache-4.2.1/src/Compression.cpp ccache-4.5.1/src/Compression.cpp --- ccache-4.2.1/src/Compression.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Compression.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Compression.hpp" - -#include "Config.hpp" -#include "Context.hpp" -#include "assertions.hpp" -#include "exceptions.hpp" - -namespace Compression { - -int8_t -level_from_config(const Config& config) -{ - return config.compression() ? config.compression_level() : 0; -} - -Type -type_from_config(const Config& config) -{ - return config.compression() ? Type::zstd : Type::none; -} - -Type -type_from_int(uint8_t type) -{ - switch (type) { - case static_cast(Type::none): - return Type::none; - - case static_cast(Type::zstd): - return Type::zstd; - } - - throw Error("Unknown type: {}", type); -} - -std::string -type_to_string(Type type) -{ - switch (type) { - case Type::none: - return "none"; - - case Type::zstd: - return "zstd"; - } - - ASSERT(false); -} - -} // namespace Compression diff -Nru ccache-4.2.1/src/Compression.hpp ccache-4.5.1/src/Compression.hpp --- ccache-4.2.1/src/Compression.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Compression.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,42 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include - -class Config; - -namespace Compression { - -enum class Type : uint8_t { - none = 0, - zstd = 1, -}; - -int8_t level_from_config(const Config& config); - -Type type_from_config(const Config& config); - -Type type_from_int(uint8_t type); - -std::string type_to_string(Compression::Type type); - -} // namespace Compression diff -Nru ccache-4.2.1/src/Compressor.cpp ccache-4.5.1/src/Compressor.cpp --- ccache-4.2.1/src/Compressor.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Compressor.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,40 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Compressor.hpp" - -#include "NullCompressor.hpp" -#include "StdMakeUnique.hpp" -#include "ZstdCompressor.hpp" -#include "assertions.hpp" - -std::unique_ptr -Compressor::create_from_type(Compression::Type type, - FILE* stream, - int8_t compression_level) -{ - switch (type) { - case Compression::Type::none: - return std::make_unique(stream); - - case Compression::Type::zstd: - return std::make_unique(stream, compression_level); - } - - ASSERT(false); -} diff -Nru ccache-4.2.1/src/Compressor.hpp ccache-4.5.1/src/Compressor.hpp --- ccache-4.2.1/src/Compressor.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Compressor.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,67 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Compression.hpp" - -#include - -class Compressor -{ -public: - virtual ~Compressor() = default; - - // Create a compressor for the specified type. - // - // Parameters: - // - type: The type. - // - stream: The stream to write to. - // - compression_level: Desired compression level. - static std::unique_ptr create_from_type(Compression::Type type, - FILE* stream, - int8_t compression_level); - - // Get the actual compression level used for the compressed stream. - virtual int8_t actual_compression_level() const = 0; - - // Write data from a buffer to the compressed stream. - // - // Parameters: - // - data: Data to write. - // - count: Size of data to write. - // - // Throws Error on failure. - virtual void write(const void* data, size_t count) = 0; - - // Write an unsigned integer to the compressed stream. - // - // Parameters: - // - value: Value to write. - // - // Throws Error on failure. - template void write(T value); - - // Finalize compression. - // - // This method checks that the end state of the compressed stream is correct - // and throws Error if not. - virtual void finalize() = 0; -}; diff -Nru ccache-4.2.1/src/Config.cpp ccache-4.5.1/src/Config.cpp --- ccache-4.2.1/src/Config.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Config.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -19,15 +19,25 @@ #include "Config.hpp" #include "AtomicFile.hpp" -#include "Compression.hpp" -#include "Sloppiness.hpp" +#include "MiniTrace.hpp" #include "Util.hpp" #include "assertions.hpp" -#include "exceptions.hpp" #include "fmtmacros.hpp" +#include +#include +#include +#include +#include +#include +#include + #include "third_party/fmt/core.h" +#ifdef HAVE_UNISTD_H +# include +#endif + #include #include #include @@ -39,6 +49,10 @@ using nonstd::nullopt; using nonstd::optional; +#ifndef environ +DLLIMPORT extern char** environ; +#endif + namespace { enum class ConfigItem { @@ -68,6 +82,7 @@ log_file, max_files, max_size, + namespace_, path, pch_external_checksum, prefix_command, @@ -75,9 +90,12 @@ read_only, read_only_direct, recache, + reshare, run_second_cpp, + secondary_storage, sloppiness, stats, + stats_log, temporary_dir, umask, }; @@ -109,6 +127,7 @@ {"log_file", ConfigItem::log_file}, {"max_files", ConfigItem::max_files}, {"max_size", ConfigItem::max_size}, + {"namespace", ConfigItem::namespace_}, {"path", ConfigItem::path}, {"pch_external_checksum", ConfigItem::pch_external_checksum}, {"prefix_command", ConfigItem::prefix_command}, @@ -116,9 +135,12 @@ {"read_only", ConfigItem::read_only}, {"read_only_direct", ConfigItem::read_only_direct}, {"recache", ConfigItem::recache}, + {"reshare", ConfigItem::reshare}, {"run_second_cpp", ConfigItem::run_second_cpp}, + {"secondary_storage", ConfigItem::secondary_storage}, {"sloppiness", ConfigItem::sloppiness}, {"stats", ConfigItem::stats}, + {"stats_log", ConfigItem::stats_log}, {"temporary_dir", ConfigItem::temporary_dir}, {"umask", ConfigItem::umask}, }; @@ -152,6 +174,7 @@ {"LOGFILE", "log_file"}, {"MAXFILES", "max_files"}, {"MAXSIZE", "max_size"}, + {"NAMESPACE", "namespace"}, {"PATH", "path"}, {"PCH_EXTSUM", "pch_external_checksum"}, {"PREFIX", "prefix_command"}, @@ -159,8 +182,11 @@ {"READONLY", "read_only"}, {"READONLY_DIRECT", "read_only_direct"}, {"RECACHE", "recache"}, + {"RESHARE", "reshare"}, + {"SECONDARY_STORAGE", "secondary_storage"}, {"SLOPPINESS", "sloppiness"}, {"STATS", "stats"}, + {"STATSLOG", "stats_log"}, {"TEMPDIR", "temporary_dir"}, {"UMASK", "umask"}, }; @@ -181,7 +207,7 @@ std::string lower_value = Util::to_lowercase(value); if (value == "0" || lower_value == "false" || lower_value == "disable" || lower_value == "no") { - throw Error( + throw core::Error( "invalid boolean environment variable value \"{}\" (did you mean to" " set \"CCACHE_{}{}=true\"?)", value, @@ -194,7 +220,7 @@ } else if (value == "false") { return false; } else { - throw Error("not a boolean value: \"{}\"", value); + throw core::Error("not a boolean value: \"{}\"", value); } } @@ -204,22 +230,6 @@ return value ? "true" : "false"; } -double -parse_double(const std::string& value) -{ - size_t end; - double result; - try { - result = std::stod(value, &end); - } catch (std::exception& e) { - throw Error(e.what()); - } - if (end != value.size()) { - throw Error("invalid floating point: \"{}\"", value); - } - return result; -} - std::string format_cache_size(uint64_t value) { @@ -245,36 +255,38 @@ } } -uint32_t +core::Sloppiness parse_sloppiness(const std::string& value) { size_t start = 0; size_t end = 0; - uint32_t result = 0; + core::Sloppiness result; while (end != std::string::npos) { end = value.find_first_of(", ", start); std::string token = - Util::strip_whitespace(value.substr(start, end - start)); + util::strip_whitespace(value.substr(start, end - start)); if (token == "file_stat_matches") { - result |= SLOPPY_FILE_STAT_MATCHES; + result.enable(core::Sloppy::file_stat_matches); } else if (token == "file_stat_matches_ctime") { - result |= SLOPPY_FILE_STAT_MATCHES_CTIME; + result.enable(core::Sloppy::file_stat_matches_ctime); } else if (token == "include_file_ctime") { - result |= SLOPPY_INCLUDE_FILE_CTIME; + result.enable(core::Sloppy::include_file_ctime); } else if (token == "include_file_mtime") { - result |= SLOPPY_INCLUDE_FILE_MTIME; + result.enable(core::Sloppy::include_file_mtime); } else if (token == "system_headers" || token == "no_system_headers") { - result |= SLOPPY_SYSTEM_HEADERS; + result.enable(core::Sloppy::system_headers); } else if (token == "pch_defines") { - result |= SLOPPY_PCH_DEFINES; + result.enable(core::Sloppy::pch_defines); } else if (token == "time_macros") { - result |= SLOPPY_TIME_MACROS; + result.enable(core::Sloppy::time_macros); } else if (token == "clang_index_store") { - result |= SLOPPY_CLANG_INDEX_STORE; + result.enable(core::Sloppy::clang_index_store); } else if (token == "locale") { - result |= SLOPPY_LOCALE; + result.enable(core::Sloppy::locale); } else if (token == "modules") { - result |= SLOPPY_MODULES; + result.enable(core::Sloppy::modules); + } else if (token == "ivfsoverlay") { + result.enable(core::Sloppy::ivfsoverlay); } // else: ignore unknown value for forward compatibility start = value.find_first_not_of(", ", end); } @@ -282,39 +294,42 @@ } std::string -format_sloppiness(uint32_t sloppiness) +format_sloppiness(core::Sloppiness sloppiness) { std::string result; - if (sloppiness & SLOPPY_INCLUDE_FILE_MTIME) { + if (sloppiness.is_enabled(core::Sloppy::include_file_mtime)) { result += "include_file_mtime, "; } - if (sloppiness & SLOPPY_INCLUDE_FILE_CTIME) { + if (sloppiness.is_enabled(core::Sloppy::include_file_ctime)) { result += "include_file_ctime, "; } - if (sloppiness & SLOPPY_TIME_MACROS) { + if (sloppiness.is_enabled(core::Sloppy::time_macros)) { result += "time_macros, "; } - if (sloppiness & SLOPPY_PCH_DEFINES) { + if (sloppiness.is_enabled(core::Sloppy::pch_defines)) { result += "pch_defines, "; } - if (sloppiness & SLOPPY_FILE_STAT_MATCHES) { + if (sloppiness.is_enabled(core::Sloppy::file_stat_matches)) { result += "file_stat_matches, "; } - if (sloppiness & SLOPPY_FILE_STAT_MATCHES_CTIME) { + if (sloppiness.is_enabled(core::Sloppy::file_stat_matches_ctime)) { result += "file_stat_matches_ctime, "; } - if (sloppiness & SLOPPY_SYSTEM_HEADERS) { + if (sloppiness.is_enabled(core::Sloppy::system_headers)) { result += "system_headers, "; } - if (sloppiness & SLOPPY_CLANG_INDEX_STORE) { + if (sloppiness.is_enabled(core::Sloppy::clang_index_store)) { result += "clang_index_store, "; } - if (sloppiness & SLOPPY_LOCALE) { + if (sloppiness.is_enabled(core::Sloppy::locale)) { result += "locale, "; } - if (sloppiness & SLOPPY_MODULES) { + if (sloppiness.is_enabled(core::Sloppy::modules)) { result += "modules, "; } + if (sloppiness.is_enabled(core::Sloppy::ivfsoverlay)) { + result += "ivfsoverlay, "; + } if (!result.empty()) { // Strip last ", ". result.resize(result.size() - 2); @@ -322,36 +337,21 @@ return result; } -uint32_t -parse_umask(const std::string& value) -{ - if (value.empty()) { - return std::numeric_limits::max(); - } - - size_t end; - uint32_t result = std::stoul(value, &end, 8); - if (end != value.size()) { - throw Error("not an octal integer: \"{}\"", value); - } - return result; -} - std::string -format_umask(uint32_t umask) +format_umask(nonstd::optional umask) { - if (umask == std::numeric_limits::max()) { - return {}; + if (umask) { + return FMT("{:03o}", *umask); } else { - return FMT("{:03o}", umask); + return {}; } } void verify_absolute_path(const std::string& value) { - if (!Util::is_absolute_path(value)) { - throw Error("not an absolute path: \"{}\"", value); + if (!util::is_absolute_path(value)) { + throw core::Error("not an absolute path: \"{}\"", value); } } @@ -361,7 +361,7 @@ std::string* value, std::string* error_message) { - std::string stripped_line = Util::strip_whitespace(line); + std::string stripped_line = util::strip_whitespace(line); if (stripped_line.empty() || stripped_line[0] == '#') { return true; } @@ -372,8 +372,8 @@ } *key = stripped_line.substr(0, equal_pos); *value = stripped_line.substr(equal_pos + 1); - *key = Util::strip_whitespace(*key); - *value = Util::strip_whitespace(*value); + *key = util::strip_whitespace(*key); + *value = util::strip_whitespace(*value); return true; } @@ -403,11 +403,11 @@ std::string value; std::string error_message; if (!parse_line(line, &key, &value, &error_message)) { - throw Error(error_message); + throw core::Error(error_message); } config_line_handler(line, key, value); - } catch (const Error& e) { - throw Error("{}:{}: {}", path, line_number, e.what()); + } catch (const core::Error& e) { + throw core::Error("{}:{}: {}", path, line_number, e.what()); } } return true; @@ -415,6 +415,30 @@ } // namespace +static std::string +default_cache_dir(const std::string& home_dir) +{ +#ifdef _WIN32 + return home_dir + "/ccache"; +#elif defined(__APPLE__) + return home_dir + "/Library/Caches/ccache"; +#else + return home_dir + "/.cache/ccache"; +#endif +} + +static std::string +default_config_dir(const std::string& home_dir) +{ +#ifdef _WIN32 + return home_dir + "/ccache"; +#elif defined(__APPLE__) + return home_dir + "/Library/Preferences/ccache"; +#else + return home_dir + "/.config/ccache"; +#endif +} + std::string compiler_type_to_string(CompilerType compiler_type) { @@ -437,6 +461,77 @@ ASSERT(false); } +void +Config::read() +{ + const std::string home_dir = Util::get_home_directory(); + const std::string legacy_ccache_dir = home_dir + "/.ccache"; + const bool legacy_ccache_dir_exists = + Stat::stat(legacy_ccache_dir).is_directory(); + const char* const env_xdg_cache_home = getenv("XDG_CACHE_HOME"); + const char* const env_xdg_config_home = getenv("XDG_CONFIG_HOME"); + + const char* env_ccache_configpath = getenv("CCACHE_CONFIGPATH"); + if (env_ccache_configpath) { + set_primary_config_path(env_ccache_configpath); + } else { + // Only used for ccache tests: + const char* const env_ccache_configpath2 = getenv("CCACHE_CONFIGPATH2"); + + set_secondary_config_path(env_ccache_configpath2 + ? env_ccache_configpath2 + : FMT("{}/ccache.conf", SYSCONFDIR)); + MTR_BEGIN("config", "conf_read_secondary"); + // A missing config file in SYSCONFDIR is OK so don't check return value. + update_from_file(secondary_config_path()); + MTR_END("config", "conf_read_secondary"); + + const char* const env_ccache_dir = getenv("CCACHE_DIR"); + std::string primary_config_dir; + if (env_ccache_dir && *env_ccache_dir) { + primary_config_dir = env_ccache_dir; + } else if (!cache_dir().empty() && !env_ccache_dir) { + primary_config_dir = cache_dir(); + } else if (legacy_ccache_dir_exists) { + primary_config_dir = legacy_ccache_dir; + } else if (env_xdg_config_home) { + primary_config_dir = FMT("{}/ccache", env_xdg_config_home); + } else { + primary_config_dir = default_config_dir(home_dir); + } + set_primary_config_path(primary_config_dir + "/ccache.conf"); + } + + const std::string& cache_dir_before_primary_config = cache_dir(); + + MTR_BEGIN("config", "conf_read_primary"); + update_from_file(primary_config_path()); + MTR_END("config", "conf_read_primary"); + + // Ignore cache_dir set in primary + set_cache_dir(cache_dir_before_primary_config); + + MTR_BEGIN("config", "conf_update_from_environment"); + update_from_environment(); + // (cache_dir is set above if CCACHE_DIR is set.) + MTR_END("config", "conf_update_from_environment"); + + if (cache_dir().empty()) { + if (legacy_ccache_dir_exists) { + set_cache_dir(legacy_ccache_dir); + } else if (env_xdg_cache_home) { + set_cache_dir(FMT("{}/ccache", env_xdg_cache_home)); + } else { + set_cache_dir(default_cache_dir(home_dir)); + } + } + // else: cache_dir was set explicitly via environment or via secondary + // config. + + // We have now determined config.cache_dir and populated the rest of config + // in prio order (1. environment, 2. primary config, 3. secondary config). +} + const std::string& Config::primary_config_path() const { @@ -464,14 +559,12 @@ bool Config::update_from_file(const std::string& path) { - return parse_config_file(path, - [&](const std::string& /*line*/, - const std::string& key, - const std::string& value) { - if (!key.empty()) { - set_item(key, value, nullopt, false, path); - } - }); + return parse_config_file( + path, [&](const auto& /*line*/, const auto& key, const auto& value) { + if (!key.empty()) { + this->set_item(key, value, nullopt, false, path); + } + }); } void @@ -480,7 +573,7 @@ for (char** env = environ; *env; ++env) { std::string setting = *env; const std::string prefix = "CCACHE_"; - if (!Util::starts_with(setting, prefix)) { + if (!util::starts_with(setting, prefix)) { continue; } size_t equal_pos = setting.find('='); @@ -490,7 +583,7 @@ std::string key = setting.substr(prefix.size(), equal_pos - prefix.size()); std::string value = setting.substr(equal_pos + 1); - bool negate = Util::starts_with(key, "NO"); + bool negate = util::starts_with(key, "NO"); if (negate) { key = key.substr(2); } @@ -504,8 +597,8 @@ try { set_item(config_key, value, key, negate, "environment"); - } catch (const Error& e) { - throw Error("CCACHE_{}{}: {}", negate ? "NO" : "", key, e.what()); + } catch (const core::Error& e) { + throw core::Error("CCACHE_{}{}: {}", negate ? "NO" : "", key, e.what()); } } } @@ -515,7 +608,7 @@ { auto it = k_config_key_table.find(key); if (it == k_config_key_table.end()) { - throw Error("unknown configuration option \"{}\"", key); + throw core::Error("unknown configuration option \"{}\"", key); } switch (it->second) { @@ -597,6 +690,9 @@ case ConfigItem::max_size: return format_cache_size(m_max_size); + case ConfigItem::namespace_: + return m_namespace; + case ConfigItem::path: return m_path; @@ -618,15 +714,24 @@ case ConfigItem::recache: return format_bool(m_recache); + case ConfigItem::reshare: + return format_bool(m_reshare); + case ConfigItem::run_second_cpp: return format_bool(m_run_second_cpp); + case ConfigItem::secondary_storage: + return m_secondary_storage; + case ConfigItem::sloppiness: return format_sloppiness(m_sloppiness); case ConfigItem::stats: return format_bool(m_stats); + case ConfigItem::stats_log: + return m_stats_log; + case ConfigItem::temporary_dir: return m_temporary_dir; @@ -640,10 +745,12 @@ void Config::set_value_in_file(const std::string& path, const std::string& key, - const std::string& value) + const std::string& value) const { + UmaskScope umask_scope(m_umask); + if (k_config_key_table.find(key) == k_config_key_table.end()) { - throw Error("unknown configuration option \"{}\"", key); + throw core::Error("unknown configuration option \"{}\"", key); } // Verify that the value is valid; set_item will throw if not. @@ -656,26 +763,25 @@ Util::ensure_dir_exists(Util::dir_name(resolved_path)); try { Util::write_file(resolved_path, ""); - } catch (const Error& e) { - throw Error("failed to write to {}: {}", resolved_path, e.what()); + } catch (const core::Error& e) { + throw core::Error("failed to write to {}: {}", resolved_path, e.what()); } } AtomicFile output(resolved_path, AtomicFile::Mode::text); bool found = false; - if (!parse_config_file(path, - [&](const std::string& c_line, - const std::string& c_key, - const std::string& /*c_value*/) { - if (c_key == key) { - output.write(FMT("{} = {}\n", key, value)); - found = true; - } else { - output.write(FMT("{}\n", c_line)); - } - })) { - throw Error("failed to open {}: {}", path, strerror(errno)); + if (!parse_config_file( + path, + [&](const auto& c_line, const auto& c_key, const auto& /*c_value*/) { + if (c_key == key) { + output.write(FMT("{} = {}\n", key, value)); + found = true; + } else { + output.write(FMT("{}\n", c_line)); + } + })) { + throw core::Error("failed to open {}: {}", path, strerror(errno)); } if (!found) { @@ -748,11 +854,10 @@ m_compression = parse_bool(value, env_var_key, negate); break; - case ConfigItem::compression_level: { - m_compression_level = - Util::parse_signed(value, INT8_MIN, INT8_MAX, "compression_level"); + case ConfigItem::compression_level: + m_compression_level = util::value_or_throw( + util::parse_signed(value, INT8_MIN, INT8_MAX, "compression_level")); break; - } case ConfigItem::cpp_extension: m_cpp_extension = value; @@ -811,7 +916,8 @@ break; case ConfigItem::limit_multiple: - m_limit_multiple = Util::clamp(parse_double(value), 0.0, 1.0); + m_limit_multiple = Util::clamp( + util::value_or_throw(util::parse_double(value)), 0.0, 1.0); break; case ConfigItem::log_file: @@ -819,13 +925,18 @@ break; case ConfigItem::max_files: - m_max_files = Util::parse_unsigned(value, nullopt, nullopt, "max_files"); + m_max_files = util::value_or_throw( + util::parse_unsigned(value, nullopt, nullopt, "max_files")); break; case ConfigItem::max_size: m_max_size = Util::parse_size(value); break; + case ConfigItem::namespace_: + m_namespace = Util::expand_environment_variables(value); + break; + case ConfigItem::path: m_path = Util::expand_environment_variables(value); break; @@ -854,10 +965,18 @@ m_recache = parse_bool(value, env_var_key, negate); break; + case ConfigItem::reshare: + m_reshare = parse_bool(value, env_var_key, negate); + break; + case ConfigItem::run_second_cpp: m_run_second_cpp = parse_bool(value, env_var_key, negate); break; + case ConfigItem::secondary_storage: + m_secondary_storage = Util::expand_environment_variables(value); + break; + case ConfigItem::sloppiness: m_sloppiness = parse_sloppiness(value); break; @@ -866,17 +985,30 @@ m_stats = parse_bool(value, env_var_key, negate); break; + case ConfigItem::stats_log: + m_stats_log = Util::expand_environment_variables(value); + break; + case ConfigItem::temporary_dir: m_temporary_dir = Util::expand_environment_variables(value); m_temporary_dir_configured_explicitly = true; break; case ConfigItem::umask: - m_umask = parse_umask(value); + if (!value.empty()) { + const auto umask = util::parse_umask(value); + if (!umask) { + throw core::Error(umask.error()); + } + m_umask = *umask; + } break; } - m_origins.emplace(key, origin); + auto result = m_origins.emplace(key, origin); + if (!result.second) { + result.first->second = origin; + } } void @@ -884,7 +1016,7 @@ { for (const auto& item : k_env_variable_table) { if (k_config_key_table.find(item.second) == k_config_key_table.end()) { - throw Error( + throw core::Error( "env var {} mapped to {} which is missing from k_config_key_table", item.first, item.second); diff -Nru ccache-4.2.1/src/Config.hpp ccache-4.5.1/src/Config.hpp --- ccache-4.2.1/src/Config.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Config.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -18,13 +18,14 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" #include "Util.hpp" +#include + #include "third_party/nonstd/optional.hpp" +#include #include #include #include @@ -34,12 +35,12 @@ std::string compiler_type_to_string(CompilerType compiler_type); -class Config +class Config : NonCopyable { public: Config() = default; - Config(Config&) = default; - Config& operator=(const Config&) = default; + + void read(); bool absolute_paths_in_stderr() const; const std::string& base_dir() const; @@ -74,20 +75,26 @@ bool read_only() const; bool read_only_direct() const; bool recache() const; + bool reshare() const; bool run_second_cpp() const; - uint32_t sloppiness() const; + const std::string& secondary_storage() const; + core::Sloppiness sloppiness() const; bool stats() const; + const std::string& stats_log() const; + const std::string& namespace_() const; const std::string& temporary_dir() const; - uint32_t umask() const; + nonstd::optional umask() const; void set_base_dir(const std::string& value); void set_cache_dir(const std::string& value); - void set_cpp_extension(const std::string& value); void set_compiler(const std::string& value); void set_compiler_type(CompilerType value); - void set_depend_mode(bool value); + void set_cpp_extension(const std::string& value); void set_debug(bool value); + void set_depend_mode(bool value); void set_direct_mode(bool value); + void set_file_clone(bool value); + void set_hard_link(bool value); void set_ignore_options(const std::string& value); void set_inode_cache(bool value); void set_max_files(uint64_t value); @@ -122,9 +129,9 @@ void visit_items(const ItemVisitor& item_visitor) const; - static void set_value_in_file(const std::string& path, - const std::string& key, - const std::string& value); + void set_value_in_file(const std::string& path, + const std::string& key, + const std::string& value) const; // Called from unit tests. static void check_key_tables_consistency(); @@ -166,11 +173,15 @@ bool m_read_only = false; bool m_read_only_direct = false; bool m_recache = false; + bool m_reshare = false; bool m_run_second_cpp = true; - uint32_t m_sloppiness = 0; + std::string m_secondary_storage; + core::Sloppiness m_sloppiness; bool m_stats = true; + std::string m_stats_log; + std::string m_namespace; std::string m_temporary_dir; - uint32_t m_umask = std::numeric_limits::max(); // Don't set umask + nonstd::optional m_umask; bool m_temporary_dir_configured_explicitly = false; @@ -384,12 +395,24 @@ } inline bool +Config::reshare() const +{ + return m_reshare; +} + +inline bool Config::run_second_cpp() const { return m_run_second_cpp; } -inline uint32_t +inline const std::string& +Config::secondary_storage() const +{ + return m_secondary_storage; +} + +inline core::Sloppiness Config::sloppiness() const { return m_sloppiness; @@ -402,12 +425,24 @@ } inline const std::string& +Config::stats_log() const +{ + return m_stats_log; +} + +inline const std::string& +Config::namespace_() const +{ + return m_namespace; +} + +inline const std::string& Config::temporary_dir() const { return m_temporary_dir; } -inline uint32_t +inline nonstd::optional Config::umask() const { return m_umask; @@ -465,6 +500,18 @@ } inline void +Config::set_file_clone(const bool value) +{ + m_file_clone = value; +} + +inline void +Config::set_hard_link(const bool value) +{ + m_hard_link = value; +} + +inline void Config::set_ignore_options(const std::string& value) { m_ignore_options = value; diff -Nru ccache-4.2.1/src/Context.cpp ccache-4.5.1/src/Context.cpp --- ccache-4.2.1/src/Context.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Context.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,12 +18,18 @@ #include "Context.hpp" -#include "Counters.hpp" #include "Logging.hpp" #include "SignalHandler.hpp" #include "Util.hpp" #include "hashutil.hpp" +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #include #include #include @@ -32,7 +38,8 @@ Context::Context() : actual_cwd(Util::get_actual_cwd()), - apparent_cwd(Util::get_apparent_cwd(actual_cwd)) + apparent_cwd(Util::get_apparent_cwd(actual_cwd)), + storage(config) #ifdef INODE_CACHE_SUPPORTED , inode_cache(config) @@ -40,6 +47,27 @@ { } +void +Context::initialize() +{ + config.read(); + Logging::init(config); + + ignore_header_paths = + util::split_path_list(config.ignore_headers_in_manifest()); + set_ignore_options(Util::split_into_strings(config.ignore_options(), " ")); + + // Set default umask for all files created by ccache from now on (if + // configured to). This is intentionally done after calling Logging::init so + // that the log file won't be affected by the umask but before creating the + // initial configuration file. The intention is that all files and directories + // in the cache directory should be affected by the configured umask and that + // no other files and directories should. + if (config.umask()) { + original_umask = umask(*config.umask()); + } +} + Context::~Context() { unlink_pending_tmp_files(); diff -Nru ccache-4.2.1/src/Context.hpp ccache-4.5.1/src/Context.hpp --- ccache-4.2.1/src/Context.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Context.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,25 +18,24 @@ #pragma once -#include "system.hpp" - #include "Args.hpp" #include "ArgsInfo.hpp" #include "Config.hpp" -#include "Counters.hpp" #include "Digest.hpp" #include "File.hpp" #include "MiniTrace.hpp" #include "NonCopyable.hpp" -#include "Sloppiness.hpp" #ifdef INODE_CACHE_SUPPORTED # include "InodeCache.hpp" #endif +#include + #include "third_party/nonstd/optional.hpp" #include "third_party/nonstd/string_view.hpp" +#include #include #include #include @@ -49,6 +48,10 @@ Context(); ~Context(); + // Read configuration, initialize logging, etc. Typically not called from unit + // tests. + void initialize(); + ArgsInfo args_info; Config config; @@ -61,20 +64,6 @@ // The original argument list. Args orig_args; - // Name (represented as a hash) of the file containing the manifest for the - // cached result. - const nonstd::optional& manifest_name() const; - - // Full path to the file containing the manifest (cachedir/a/b/cdef[...]M), if - // any. - const nonstd::optional& manifest_path() const; - - // Name (represented as a hash) of the file containing the cached result. - const nonstd::optional& result_name() const; - - // Full path to the file containing the result (cachedir/a/b/cdef[...]R). - const nonstd::optional& result_path() const; - // Time of compilation. Used to see if include files have changed after // compilation. time_t time_of_compilation = 0; @@ -94,25 +83,17 @@ // The name of the cpp stderr file. std::string cpp_stderr; - // The .gch/.pch/.pth file used for compilation. - std::string included_pch_file; - // Headers (or directories with headers) to ignore in manifest mode. std::vector ignore_header_paths; + // Storage (fronting primary and secondary storage backends). + storage::Storage storage; + #ifdef INODE_CACHE_SUPPORTED // InodeCache that caches source file hashes when enabled. mutable InodeCache inode_cache; #endif - // Statistics updates which get written into the statistics file belonging to - // the result. - Counters counter_updates; - - // Statistics updates which get written into the statistics file belonging to - // the manifest. - Counters manifest_counter_updates; - // PID of currently executing compiler that we have started, if any. 0 means // no ongoing compilation. pid_t compiler_pid = 0; @@ -133,21 +114,10 @@ std::unique_ptr mini_trace; #endif - void set_manifest_name(const Digest& name); - void set_manifest_path(const std::string& path); - void set_result_name(const Digest& name); - void set_result_path(const std::string& path); - // Register a temporary file to remove at program exit. void register_pending_tmp_file(const std::string& path); private: - nonstd::optional m_manifest_name; - nonstd::optional m_manifest_path; - - nonstd::optional m_result_name; - nonstd::optional m_result_path; - // Options to ignore for the hash. std::vector m_ignore_options; @@ -163,56 +133,8 @@ void unlink_pending_tmp_files_signal_safe(); // called from signal handler }; -inline const nonstd::optional& -Context::manifest_name() const -{ - return m_manifest_name; -} - -inline const nonstd::optional& -Context::manifest_path() const -{ - return m_manifest_path; -} - -inline const nonstd::optional& -Context::result_name() const -{ - return m_result_name; -} - -inline const nonstd::optional& -Context::result_path() const -{ - return m_result_path; -} - inline const std::vector& Context::ignore_options() const { return m_ignore_options; } - -inline void -Context::set_manifest_name(const Digest& name) -{ - m_manifest_name = name; -} - -inline void -Context::set_manifest_path(const std::string& path) -{ - m_manifest_path = path; -} - -inline void -Context::set_result_name(const Digest& name) -{ - m_result_name = name; -} - -inline void -Context::set_result_path(const std::string& path) -{ - m_result_path = path; -} diff -Nru ccache-4.2.1/src/core/CacheEntryHeader.cpp ccache-4.5.1/src/core/CacheEntryHeader.cpp --- ccache-4.2.1/src/core/CacheEntryHeader.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/CacheEntryHeader.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,95 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheEntryHeader.hpp" + +#include + +const size_t k_static_header_fields_size = + sizeof(core::CacheEntryHeader::magic) + + sizeof(core::CacheEntryHeader::entry_format_version) + + sizeof(core::CacheEntryHeader::entry_type) + + sizeof(core::CacheEntryHeader::compression_type) + + sizeof(core::CacheEntryHeader::compression_level) + + sizeof(core::CacheEntryHeader::creation_time) + + sizeof(core::CacheEntryHeader::entry_size) + // ccache_version length field: + + 1 + // namespace_ length field: + + 1; + +const size_t k_static_epilogue_fields_size = + sizeof(uint64_t) + sizeof(uint64_t); + +namespace core { + +CacheEntryHeader::CacheEntryHeader(const core::CacheEntryType entry_type_, + const compression::Type compression_type_, + const int8_t compression_level_, + const uint64_t creation_time_, + const std::string& ccache_version_, + const std::string& namespace_arg, + const uint64_t entry_size_) + : magic(k_ccache_magic), + entry_format_version(k_entry_format_version), + entry_type(entry_type_), + compression_type(compression_type_), + compression_level(compression_level_), + creation_time(creation_time_), + ccache_version(ccache_version_), + namespace_(namespace_arg), + entry_size(entry_size_) +{ +} + +uint64_t +CacheEntryHeader::payload_size() const +{ + return entry_size - non_payload_size(); +} + +void +CacheEntryHeader::set_entry_size_from_payload_size(const uint64_t payload_size) +{ + entry_size = non_payload_size() + payload_size; +} + +void +CacheEntryHeader::dump(FILE* const stream) const +{ + PRINT(stream, "Magic: {:04x}\n", magic); + PRINT(stream, "Entry format version: {}\n", entry_format_version); + PRINT(stream, "Entry type: {} ({})\n", entry_type, to_string(entry_type)); + PRINT(stream, + "Compression type: {}\n", + compression::type_to_string(compression_type)); + PRINT(stream, "Compression level: {}\n", compression_level); + PRINT(stream, "Creation time: {}\n", creation_time); + PRINT(stream, "Ccache version: {}\n", ccache_version); + PRINT(stream, "Namespace: {}\n", namespace_); + PRINT(stream, "Entry size: {}\n", entry_size); +} + +size_t +CacheEntryHeader::non_payload_size() const +{ + return k_static_header_fields_size + ccache_version.length() + + namespace_.length() + k_static_epilogue_fields_size; +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/CacheEntryHeader.hpp ccache-4.5.1/src/core/CacheEntryHeader.hpp --- ccache-4.2.1/src/core/CacheEntryHeader.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/CacheEntryHeader.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,85 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +// Cache entry format +// ================== +// +// Integers are big-endian. +// +// ::=
+//
::= +// +// +// ::= uint16_t (0xccac) +// ::= uint8_t +// ::= | +// ::= 0 (uint8_t) +// ::= 1 (uint8_t) +// ::= | +// ::= 0 (uint8_t) +// ::= 1 (uint8_t) +// ::= int8_t +// ::= uint64_t (Unix epoch time when entry was created) +// ::= string length (uint8_t) + string data +// ::= string length (uint8_t) + string data +// ::= uint64_t ; = size of file if stored uncompressed +// ; potentially compressed from here +// ::= depends on entry_type +// ::= +// ::= uint64_t ; XXH3-128 (high bits) of entry bytes +// ::= uint64_t ; XXH3-128 (low bits) of entry bytes + +namespace core { + +const uint16_t k_ccache_magic = 0xccac; +const uint16_t k_entry_format_version = 0; + +struct CacheEntryHeader +{ + CacheEntryHeader(core::CacheEntryType entry_type, + compression::Type compression_type, + int8_t compression_level, + uint64_t creation_time, + const std::string& ccache_version, + const std::string& namespace_, + uint64_t entry_size = 0); + + uint16_t magic; + uint8_t entry_format_version; + core::CacheEntryType entry_type; + compression::Type compression_type; + int8_t compression_level; + uint64_t creation_time; + std::string ccache_version; + std::string namespace_; + uint64_t entry_size; + + uint64_t payload_size() const; + void set_entry_size_from_payload_size(uint64_t payload_size); + void dump(FILE* stream) const; + +private: + size_t non_payload_size() const; +}; + +} // namespace core diff -Nru ccache-4.2.1/src/core/CacheEntryReader.cpp ccache-4.5.1/src/core/CacheEntryReader.cpp --- ccache-4.2.1/src/core/CacheEntryReader.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/CacheEntryReader.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,108 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheEntryReader.hpp" + +#include + +namespace { + +core::CacheEntryType +cache_entry_type_from_int(const uint8_t entry_type) +{ + switch (entry_type) { + case 0: + return core::CacheEntryType::result; + break; + case 1: + return core::CacheEntryType::manifest; + break; + default: + throw core::Error("Unknown entry type: {}", entry_type); + } +} + +} // namespace + +namespace core { + +CacheEntryReader::CacheEntryReader(core::Reader& reader) + : m_checksumming_reader(reader) +{ + const auto magic = m_checksumming_reader.read_int(); + if (magic != core::k_ccache_magic) { + throw core::Error("Bad magic value: 0x{:04x}", magic); + } + + const auto entry_format_version = m_checksumming_reader.read_int(); + if (entry_format_version != core::k_entry_format_version) { + throw core::Error("Unknown entry format version: {}", entry_format_version); + } + + const auto entry_type = m_checksumming_reader.read_int(); + const auto compression_type = m_checksumming_reader.read_int(); + const auto compression_level = m_checksumming_reader.read_int(); + const auto creation_time = m_checksumming_reader.read_int(); + const auto ccache_version = + m_checksumming_reader.read_str(m_checksumming_reader.read_int()); + const auto tag = + m_checksumming_reader.read_str(m_checksumming_reader.read_int()); + const auto entry_size = m_checksumming_reader.read_int(); + + m_header = std::make_unique( + cache_entry_type_from_int(entry_type), + compression::type_from_int(compression_type), + compression_level, + creation_time, + ccache_version, + tag, + entry_size); + + m_decompressor = compression::Decompressor::create_from_type( + m_header->compression_type, reader); + m_checksumming_reader.set_reader(*m_decompressor); +} + +size_t +CacheEntryReader::read(void* const data, const size_t count) +{ + return m_checksumming_reader.read(data, count); +} + +void +CacheEntryReader::finalize() +{ + const util::XXH3_128::Digest actual = m_checksumming_reader.digest(); + util::XXH3_128::Digest expected; + m_decompressor->read(expected.bytes(), expected.size()); + + // actual == null_digest: Checksumming is not enabled now. + // expected == null_digest: Checksumming was not enabled when the entry was + // created. + const util::XXH3_128::Digest null_digest; + + if (actual != expected && actual != null_digest && expected != null_digest) { + throw core::Error("Incorrect checksum (actual {}, expected {})", + Util::format_base16(actual.bytes(), actual.size()), + Util::format_base16(expected.bytes(), expected.size())); + } + + m_decompressor->finalize(); +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/CacheEntryReader.hpp ccache-4.5.1/src/core/CacheEntryReader.hpp --- ccache-4.2.1/src/core/CacheEntryReader.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/CacheEntryReader.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,61 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include +#include + +namespace core { + +// This class knows how to read a cache entry with a format described in +// CacheEntryHeader. +class CacheEntryReader : public Reader +{ +public: + // Read cache entry data from `reader`. + CacheEntryReader(Reader& reader); + + size_t read(void* data, size_t count) override; + using Reader::read; + + // Close for reading. + // + // This method potentially verifies the end state after reading the cache + // entry and throws `core::Error` if any integrity issues are found. + void finalize(); + + const CacheEntryHeader& header() const; + +private: + ChecksummingReader m_checksumming_reader; + std::unique_ptr m_header; + util::XXH3_128 m_checksum; + std::unique_ptr m_decompressor; +}; + +inline const CacheEntryHeader& +CacheEntryReader::header() const +{ + return *m_header; +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/CacheEntryWriter.cpp ccache-4.5.1/src/core/CacheEntryWriter.cpp --- ccache-4.2.1/src/core/CacheEntryWriter.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/CacheEntryWriter.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,61 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheEntryWriter.hpp" + +#include + +namespace core { + +CacheEntryWriter::CacheEntryWriter(core::Writer& writer, + const CacheEntryHeader& header) + : m_checksumming_writer(writer), + m_compressor(compression::Compressor::create_from_type( + header.compression_type, writer, header.compression_level)) +{ + m_checksumming_writer.write_int(header.magic); + m_checksumming_writer.write_int(header.entry_format_version); + m_checksumming_writer.write_int(static_cast(header.entry_type)); + m_checksumming_writer.write_int( + static_cast(header.compression_type)); + m_checksumming_writer.write_int(m_compressor->actual_compression_level()); + m_checksumming_writer.write_int(header.creation_time); + m_checksumming_writer.write_int(header.ccache_version.length()); + m_checksumming_writer.write_str(header.ccache_version); + m_checksumming_writer.write_int(header.namespace_.length()); + m_checksumming_writer.write_str(header.namespace_); + m_checksumming_writer.write_int(header.entry_size); + + m_checksumming_writer.set_writer(*m_compressor); +} + +void +CacheEntryWriter::write(const void* const data, const size_t count) +{ + m_checksumming_writer.write(data, count); +} + +void +CacheEntryWriter::finalize() +{ + const auto digest = m_checksumming_writer.digest(); + m_compressor->write(digest.bytes(), digest.size()); + m_compressor->finalize(); +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/CacheEntryWriter.hpp ccache-4.5.1/src/core/CacheEntryWriter.hpp --- ccache-4.2.1/src/core/CacheEntryWriter.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/CacheEntryWriter.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,50 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include + +namespace core { + +struct CacheEntryHeader; + +// This class knows how to write a cache entry with a format described in +// CacheEntryHeader. +class CacheEntryWriter : public Writer +{ +public: + CacheEntryWriter(Writer& writer, const CacheEntryHeader& header); + + void write(const void* data, size_t count) override; + using Writer::write; + + // Close for writing. + // + // This method potentially verifies the end state after writing the cache + // entry and throws `core::Error` if any integrity issues are found. + void finalize() override; + +private: + ChecksummingWriter m_checksumming_writer; + std::unique_ptr m_compressor; +}; + +} // namespace core diff -Nru ccache-4.2.1/src/core/ChecksummingReader.hpp ccache-4.5.1/src/core/ChecksummingReader.hpp --- ccache-4.2.1/src/core/ChecksummingReader.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/ChecksummingReader.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,68 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace core { + +class ChecksummingReader : public Reader +{ +public: + ChecksummingReader(core::Reader& reader); + + using core::Reader::read; + size_t read(void* data, size_t count) override; + + void set_reader(core::Reader& reader); + + util::XXH3_128::Digest digest() const; + +private: + core::Reader* m_reader; + util::XXH3_128 m_checksum; +}; + +inline ChecksummingReader::ChecksummingReader(core::Reader& reader) + : m_reader(&reader) +{ +} + +inline size_t +ChecksummingReader::read(void* const data, const size_t count) +{ + const auto bytes_read = m_reader->read(data, count); + m_checksum.update(data, bytes_read); + return bytes_read; +} + +inline void +ChecksummingReader::set_reader(core::Reader& reader) +{ + m_reader = &reader; +} + +inline util::XXH3_128::Digest +ChecksummingReader::digest() const +{ + return m_checksum.digest(); +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/ChecksummingWriter.hpp ccache-4.5.1/src/core/ChecksummingWriter.hpp --- ccache-4.2.1/src/core/ChecksummingWriter.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/ChecksummingWriter.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,74 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace core { + +class ChecksummingWriter : public Writer +{ +public: + ChecksummingWriter(core::Writer& writer); + + using core::Writer::write; + void write(const void* data, size_t count) override; + void finalize() override; + + void set_writer(core::Writer& writer); + + util::XXH3_128::Digest digest() const; + +private: + core::Writer* m_writer; + util::XXH3_128 m_checksum; +}; + +inline ChecksummingWriter::ChecksummingWriter(core::Writer& writer) + : m_writer(&writer) +{ +} + +inline void +ChecksummingWriter::write(const void* const data, const size_t count) +{ + m_writer->write(data, count); + m_checksum.update(data, count); +} + +inline void +ChecksummingWriter::finalize() +{ + m_writer->finalize(); +} + +inline void +ChecksummingWriter::set_writer(core::Writer& writer) +{ + m_writer = &writer; +} + +inline util::XXH3_128::Digest +ChecksummingWriter::digest() const +{ + return m_checksum.digest(); +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/CMakeLists.txt ccache-4.5.1/src/core/CMakeLists.txt --- ccache-4.2.1/src/core/CMakeLists.txt 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,13 @@ +set( + sources + ${CMAKE_CURRENT_SOURCE_DIR}/CacheEntryHeader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/CacheEntryReader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/CacheEntryWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Statistics.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/StatisticsCounters.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/StatsLog.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/mainoptions.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/types.cpp +) + +target_sources(ccache_framework PRIVATE ${sources}) diff -Nru ccache-4.2.1/src/core/exceptions.hpp ccache-4.5.1/src/core/exceptions.hpp --- ccache-4.2.1/src/core/exceptions.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/exceptions.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,83 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include + +#include +#include +#include + +namespace core { + +// Don't throw or catch ErrorBase directly, use a subclass. +class ErrorBase : public std::runtime_error +{ + using std::runtime_error::runtime_error; +}; + +// Throw an Error to indicate a potentially non-fatal error that may be caught +// and handled by callers. An uncaught Error that reaches the top level will be +// treated similar to Fatal. +class Error : public ErrorBase +{ +public: + // Special case: If given only one string, don't parse it as a format string. + Error(const std::string& message); + + // `args` are forwarded to `fmt::format`. + template inline Error(T&&... args); +}; + +inline Error::Error(const std::string& message) : ErrorBase(message) +{ +} + +template +inline Error::Error(T&&... args) + : ErrorBase(fmt::format(std::forward(args)...)) +{ +} + +// Throw a Fatal to make ccache print the error message to stderr and exit +// with a non-zero exit code. +class Fatal : public ErrorBase +{ +public: + // Special case: If given only one string, don't parse it as a format string. + Fatal(const std::string& message); + + // `args` are forwarded to `fmt::format`. + template inline Fatal(T&&... args); +}; + +inline Fatal::Fatal(const std::string& message) : ErrorBase(message) +{ +} + +template +inline Fatal::Fatal(T&&... args) + : ErrorBase(fmt::format(std::forward(args)...)) +{ +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/FileReader.hpp ccache-4.5.1/src/core/FileReader.hpp --- ccache-4.2.1/src/core/FileReader.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/FileReader.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,56 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include + +namespace core { + +class FileReader : public Reader +{ +public: + FileReader(FILE* stream); + + size_t read(void* data, size_t size) override; + +private: + FILE* m_stream; +}; + +inline FileReader::FileReader(FILE* stream) : m_stream(stream) +{ +} + +inline size_t +FileReader::read(void* const data, const size_t size) +{ + if (size == 0) { + return 0; + } + const auto bytes_read = fread(data, 1, size, m_stream); + if (bytes_read == 0) { + throw core::Error("Failed to read from file stream"); + } + return bytes_read; +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/FileWriter.hpp ccache-4.5.1/src/core/FileWriter.hpp --- ccache-4.2.1/src/core/FileWriter.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/FileWriter.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,58 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include + +namespace core { + +class FileWriter : public Writer +{ +public: + FileWriter(FILE* stream); + + void write(const void* data, size_t size) override; + void finalize() override; + +private: + FILE* m_stream; +}; + +inline FileWriter::FileWriter(FILE* const stream) : m_stream(stream) +{ +} + +inline void +FileWriter::write(const void* const data, const size_t size) +{ + if (size > 0 && fwrite(data, size, 1, m_stream) != 1) { + throw core::Error("Failed to write to stream"); + } +} + +inline void +FileWriter::finalize() +{ + fflush(m_stream); +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/mainoptions.cpp ccache-4.5.1/src/core/mainoptions.cpp --- ccache-4.2.1/src/core/mainoptions.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/mainoptions.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,615 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "mainoptions.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +#ifdef HAVE_GETOPT_LONG +# include +#elif defined(_WIN32) +# include +#else +extern "C" { +# include +} +#endif + +namespace core { + +constexpr const char VERSION_TEXT[] = + R"({0} version {1} +Features: {2} + +Copyright (C) 2002-2007 Andrew Tridgell +Copyright (C) 2009-2021 Joel Rosdahl and other contributors + +See for a complete list of contributors. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. +)"; + +constexpr const char USAGE_TEXT[] = + R"(Usage: + {0} [options] + {0} compiler [compiler options] + compiler [compiler options] (via symbolic link) + +Common options: + -c, --cleanup delete old files and recalculate size counters + (normally not needed as this is done + automatically) + -C, --clear clear the cache completely (except configuration) + --config-path PATH operate on configuration file PATH instead of the + default + -d, --dir PATH operate on cache directory PATH instead of the + default + --evict-namespace NAMESPACE + remove files created in namespace NAMESPACE + --evict-older-than AGE remove files older than AGE (unsigned integer + with a d (days) or s (seconds) suffix) + -F, --max-files NUM set maximum number of files in cache to NUM (use + 0 for no limit) + -M, --max-size SIZE set maximum size of cache to SIZE (use 0 for no + limit); available suffixes: k, M, G, T (decimal) + and Ki, Mi, Gi, Ti (binary); default suffix: G + -X, --recompress LEVEL recompress the cache to level LEVEL (integer or + "uncompressed") using the Zstandard algorithm; + see "Cache compression" in the manual for details + -o, --set-config KEY=VAL set configuration item KEY to value VAL + -x, --show-compression show compression statistics + -p, --show-config show current configuration options in + human-readable format + --show-log-stats print statistics counters from the stats log + in human-readable format + -s, --show-stats show summary of configuration and statistics + counters in human-readable format + -v, --verbose increase verbosity + -z, --zero-stats zero statistics counters + + -h, --help print this help text + -V, --version print version and copyright information + +Options for secondary storage: + --trim-dir PATH remove old files from directory _PATH_ until it + is at most the size specified by --trim-max-size + (note: don't use this option to trim the primary + cache) + --trim-max-size SIZE specify the maximum size for --trim-dir; + available suffixes: k, M, G, T (decimal) and Ki, + Mi, Gi, Ti (binary); default suffix: G + --trim-method METHOD specify the method (atime or mtime) for + --trim-dir; default: atime + +Options for scripting or debugging: + --checksum-file PATH print the checksum (128 bit XXH3) of the file at + PATH + --dump-manifest PATH dump manifest file at PATH in text format + --dump-result PATH dump result file at PATH in text format + --extract-result PATH extract data stored in result file at PATH to the + current working directory + -k, --get-config KEY print the value of configuration key KEY + --hash-file PATH print the hash (160 bit BLAKE3) of the file at + PATH + --print-stats print statistics counter IDs and corresponding + values in machine-parsable format + +See also the manual on . +)"; + +static void +configuration_printer(const std::string& key, + const std::string& value, + const std::string& origin) +{ + PRINT(stdout, "({}) {} = {}\n", origin, key, value); +} + +static void +print_compression_statistics(const storage::primary::CompressionStatistics& cs) +{ + const double ratio = cs.compr_size > 0 + ? static_cast(cs.content_size) / cs.compr_size + : 0.0; + const double savings = ratio > 0.0 ? 100.0 - (100.0 / ratio) : 0.0; + + using C = util::TextTable::Cell; + auto human_readable = Util::format_human_readable_size; + util::TextTable table; + + table.add_row({ + "Total data:", + C(human_readable(cs.compr_size + cs.incompr_size)).right_align(), + FMT("({} disk blocks)", human_readable(cs.on_disk_size)), + }); + table.add_row({ + "Compressed data:", + C(human_readable(cs.compr_size)).right_align(), + FMT("({:.1f}% of original size)", 100.0 - savings), + }); + table.add_row({ + " Original size:", + C(human_readable(cs.content_size)).right_align(), + }); + table.add_row({ + " Compression ratio:", + C(FMT("{:.3f} x ", ratio)).right_align(), + FMT("({:.1f}% space savings)", savings), + }); + table.add_row({ + "Incompressible data:", + C(human_readable(cs.incompr_size)).right_align(), + }); + + PRINT_RAW(stdout, table.render()); +} + +static void +trim_dir(const std::string& dir, + const uint64_t trim_max_size, + const bool trim_lru_mtime) +{ + struct File + { + std::string path; + Stat stat; + }; + std::vector files; + uint64_t size_before = 0; + + Util::traverse(dir, [&](const std::string& path, const bool is_dir) { + const auto stat = Stat::lstat(path); + if (!stat) { + // Probably some race, ignore. + return; + } + size_before += stat.size_on_disk(); + if (!is_dir) { + const auto name = Util::base_name(path); + if (name == "ccache.conf" || name == "stats") { + throw Fatal("this looks like a primary cache directory (found {})", + path); + } + files.push_back({path, stat}); + } + }); + + std::sort(files.begin(), files.end(), [&](const auto& f1, const auto& f2) { + const auto ts_1 = trim_lru_mtime ? f1.stat.mtim() : f1.stat.atim(); + const auto ts_2 = trim_lru_mtime ? f2.stat.mtim() : f2.stat.atim(); + const auto ns_1 = 1'000'000'000ULL * ts_1.tv_sec + ts_1.tv_nsec; + const auto ns_2 = 1'000'000'000ULL * ts_2.tv_sec + ts_2.tv_nsec; + return ns_1 < ns_2; + }); + + uint64_t size_after = size_before; + + for (const auto& file : files) { + if (size_after <= trim_max_size) { + break; + } + Util::unlink_tmp(file.path); + size_after -= file.stat.size(); + } + + PRINT(stdout, + "Removed {} ({} -> {})\n", + Util::format_human_readable_size(size_before - size_after), + Util::format_human_readable_size(size_before), + Util::format_human_readable_size(size_after)); +} + +static std::string +get_version_text() +{ + return FMT( + VERSION_TEXT, CCACHE_NAME, CCACHE_VERSION, storage::get_features()); +} + +std::string +get_usage_text() +{ + return FMT(USAGE_TEXT, CCACHE_NAME); +} + +enum { + CHECKSUM_FILE, + CONFIG_PATH, + DUMP_MANIFEST, + DUMP_RESULT, + EVICT_NAMESPACE, + EVICT_OLDER_THAN, + EXTRACT_RESULT, + HASH_FILE, + PRINT_STATS, + SHOW_LOG_STATS, + TRIM_DIR, + TRIM_MAX_SIZE, + TRIM_METHOD, +}; + +const char options_string[] = "cCd:k:hF:M:po:svVxX:z"; +const option long_options[] = { + {"checksum-file", required_argument, nullptr, CHECKSUM_FILE}, + {"cleanup", no_argument, nullptr, 'c'}, + {"clear", no_argument, nullptr, 'C'}, + {"config-path", required_argument, nullptr, CONFIG_PATH}, + {"dir", required_argument, nullptr, 'd'}, + {"directory", required_argument, nullptr, 'd'}, // backward compatibility + {"dump-manifest", required_argument, nullptr, DUMP_MANIFEST}, + {"dump-result", required_argument, nullptr, DUMP_RESULT}, + {"evict-namespace", required_argument, nullptr, EVICT_NAMESPACE}, + {"evict-older-than", required_argument, nullptr, EVICT_OLDER_THAN}, + {"extract-result", required_argument, nullptr, EXTRACT_RESULT}, + {"get-config", required_argument, nullptr, 'k'}, + {"hash-file", required_argument, nullptr, HASH_FILE}, + {"help", no_argument, nullptr, 'h'}, + {"max-files", required_argument, nullptr, 'F'}, + {"max-size", required_argument, nullptr, 'M'}, + {"print-stats", no_argument, nullptr, PRINT_STATS}, + {"recompress", required_argument, nullptr, 'X'}, + {"set-config", required_argument, nullptr, 'o'}, + {"show-compression", no_argument, nullptr, 'x'}, + {"show-config", no_argument, nullptr, 'p'}, + {"show-log-stats", no_argument, nullptr, SHOW_LOG_STATS}, + {"show-stats", no_argument, nullptr, 's'}, + {"trim-dir", required_argument, nullptr, TRIM_DIR}, + {"trim-max-size", required_argument, nullptr, TRIM_MAX_SIZE}, + {"trim-method", required_argument, nullptr, TRIM_METHOD}, + {"verbose", no_argument, nullptr, 'v'}, + {"version", no_argument, nullptr, 'V'}, + {"zero-stats", no_argument, nullptr, 'z'}, + {nullptr, 0, nullptr, 0}}; + +int +process_main_options(int argc, const char* const* argv) +{ + int c; + nonstd::optional trim_max_size; + bool trim_lru_mtime = false; + uint8_t verbosity = 0; + nonstd::optional evict_namespace; + nonstd::optional evict_max_age; + + // First pass: Handle non-command options that affect command options. + while ((c = getopt_long(argc, + const_cast(argv), + options_string, + long_options, + nullptr)) + != -1) { + const std::string arg = optarg ? optarg : std::string(); + + switch (c) { + case 'd': // --dir + Util::setenv("CCACHE_DIR", arg); + break; + + case CONFIG_PATH: + Util::setenv("CCACHE_CONFIGPATH", arg); + break; + + case TRIM_MAX_SIZE: + trim_max_size = Util::parse_size(arg); + break; + + case TRIM_METHOD: + trim_lru_mtime = (arg == "ctime"); + break; + + case 'v': // --verbose + ++verbosity; + break; + + case '?': // unknown option + return EXIT_FAILURE; + } + } + + // Second pass: Handle command options in order. + optind = 1; + while ((c = getopt_long(argc, + const_cast(argv), + options_string, + long_options, + nullptr)) + != -1) { + Config config; + config.read(); + + const std::string arg = optarg ? optarg : std::string(); + + switch (c) { + case CONFIG_PATH: + case 'd': // --dir + case TRIM_MAX_SIZE: + case TRIM_METHOD: + case 'v': // --verbose + // Already handled in the first pass. + break; + + case CHECKSUM_FILE: { + util::XXH3_128 checksum; + Fd fd(arg == "-" ? STDIN_FILENO : open(arg.c_str(), O_RDONLY)); + Util::read_fd(*fd, [&checksum](const void* data, size_t size) { + checksum.update(data, size); + }); + const auto digest = checksum.digest(); + PRINT(stdout, "{}\n", Util::format_base16(digest.bytes(), digest.size())); + break; + } + + case DUMP_MANIFEST: + return Manifest::dump(arg, stdout) ? 0 : 1; + + case DUMP_RESULT: { + ResultDumper result_dumper(stdout); + Result::Reader result_reader(arg); + auto error = result_reader.read(result_dumper); + if (error) { + PRINT(stderr, "Error: {}\n", *error); + } + return error ? EXIT_FAILURE : EXIT_SUCCESS; + } + + case EVICT_NAMESPACE: { + evict_namespace = arg; + break; + } + + case EVICT_OLDER_THAN: { + evict_max_age = Util::parse_duration(arg); + break; + } + + case EXTRACT_RESULT: { + ResultExtractor result_extractor("."); + Result::Reader result_reader(arg); + auto error = result_reader.read(result_extractor); + if (error) { + PRINT(stderr, "Error: {}\n", *error); + } + return error ? EXIT_FAILURE : EXIT_SUCCESS; + } + + case HASH_FILE: { + Hash hash; + if (arg == "-") { + hash.hash_fd(STDIN_FILENO); + } else { + hash.hash_file(arg); + } + PRINT(stdout, "{}\n", hash.digest().to_string()); + break; + } + + case PRINT_STATS: { + StatisticsCounters counters; + time_t last_updated; + std::tie(counters, last_updated) = + storage::primary::PrimaryStorage(config).get_all_statistics(); + Statistics statistics(counters); + PRINT_RAW(stdout, statistics.format_machine_readable(last_updated)); + break; + } + + case 'c': // --cleanup + { + ProgressBar progress_bar("Cleaning..."); + storage::primary::PrimaryStorage(config).clean_all( + [&](double progress) { progress_bar.update(progress); }); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n"); + } + break; + } + + case 'C': // --clear + { + ProgressBar progress_bar("Clearing..."); + storage::primary::PrimaryStorage(config).wipe_all( + [&](double progress) { progress_bar.update(progress); }); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n"); + } +#ifdef INODE_CACHE_SUPPORTED + InodeCache(config).drop(); +#endif + break; + } + + case 'h': // --help + PRINT(stdout, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); + return EXIT_SUCCESS; + + case 'k': // --get-config + PRINT(stdout, "{}\n", config.get_string_value(arg)); + break; + + case 'F': { // --max-files + auto files = util::value_or_throw(util::parse_unsigned(arg)); + config.set_value_in_file(config.primary_config_path(), "max_files", arg); + if (files == 0) { + PRINT_RAW(stdout, "Unset cache file limit\n"); + } else { + PRINT(stdout, "Set cache file limit to {}\n", files); + } + break; + } + + case 'M': { // --max-size + uint64_t size = Util::parse_size(arg); + config.set_value_in_file(config.primary_config_path(), "max_size", arg); + if (size == 0) { + PRINT_RAW(stdout, "Unset cache size limit\n"); + } else { + PRINT(stdout, + "Set cache size limit to {}\n", + Util::format_human_readable_size(size)); + } + break; + } + + case 'o': { // --set-config + // Start searching for equal sign at position 1 to improve error message + // for the -o=K=V case (key "=K" and value "V"). + size_t eq_pos = arg.find('=', 1); + if (eq_pos == std::string::npos) { + throw Error("missing equal sign in \"{}\"", arg); + } + std::string key = arg.substr(0, eq_pos); + std::string value = arg.substr(eq_pos + 1); + config.set_value_in_file(config.primary_config_path(), key, value); + break; + } + + case 'p': // --show-config + config.visit_items(configuration_printer); + break; + + case SHOW_LOG_STATS: { + if (config.stats_log().empty()) { + throw Fatal("No stats log has been configured"); + } + Statistics statistics(StatsLog(config.stats_log()).read()); + const auto timestamp = + Stat::stat(config.stats_log(), Stat::OnError::log).mtime(); + PRINT_RAW( + stdout, + statistics.format_human_readable(config, timestamp, verbosity, true)); + if (verbosity == 0) { + PRINT_RAW(stdout, "\nUse the -v/--verbose option for more details.\n"); + } + break; + } + + case 's': { // --show-stats + StatisticsCounters counters; + time_t last_updated; + std::tie(counters, last_updated) = + storage::primary::PrimaryStorage(config).get_all_statistics(); + Statistics statistics(counters); + PRINT_RAW(stdout, + statistics.format_human_readable( + config, last_updated, verbosity, false)); + if (verbosity == 0) { + PRINT_RAW(stdout, "\nUse the -v/--verbose option for more details.\n"); + } + break; + } + + case TRIM_DIR: + if (!trim_max_size) { + throw Error("please specify --trim-max-size when using --trim-dir"); + } + trim_dir(arg, *trim_max_size, trim_lru_mtime); + break; + + case 'V': // --version + PRINT_RAW(stdout, get_version_text()); + break; + + case 'x': // --show-compression + { + ProgressBar progress_bar("Scanning..."); + const auto compression_statistics = + storage::primary::PrimaryStorage(config).get_compression_statistics( + [&](double progress) { progress_bar.update(progress); }); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n\n"); + } + print_compression_statistics(compression_statistics); + break; + } + + case 'X': // --recompress + { + nonstd::optional wanted_level; + if (arg == "uncompressed") { + wanted_level = nonstd::nullopt; + } else { + wanted_level = util::value_or_throw( + util::parse_signed(arg, INT8_MIN, INT8_MAX, "compression level")); + } + + ProgressBar progress_bar("Recompressing..."); + storage::primary::PrimaryStorage(config).recompress( + wanted_level, [&](double progress) { progress_bar.update(progress); }); + break; + } + + case 'z': // --zero-stats + storage::primary::PrimaryStorage(config).zero_all_statistics(); + PRINT_RAW(stdout, "Statistics zeroed\n"); + break; + + default: + PRINT(stderr, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); + return EXIT_FAILURE; + } + } + + if (evict_max_age || evict_namespace) { + Config config; + config.read(); + + ProgressBar progress_bar("Evicting..."); + storage::primary::PrimaryStorage(config).evict( + [&](double progress) { progress_bar.update(progress); }, + evict_max_age, + evict_namespace); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n"); + } + } + + return EXIT_SUCCESS; +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/mainoptions.hpp ccache-4.5.1/src/core/mainoptions.hpp --- ccache-4.2.1/src/core/mainoptions.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/mainoptions.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,30 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace core { + +// The main program when not doing a compile. +int process_main_options(int argc, const char* const* argv); + +std::string get_usage_text(); + +} // namespace core diff -Nru ccache-4.2.1/src/core/Reader.hpp ccache-4.5.1/src/core/Reader.hpp --- ccache-4.2.1/src/core/Reader.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/Reader.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,79 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include +#include +#include + +namespace core { + +class Reader +{ +public: + virtual ~Reader() = default; + + // Read `count` bytes into `data`, returning the actual number of bytes read + // if not enough data is available. Throws `core::Error` on failure, e.g. if + // no bytes could be read. + virtual size_t read(void* data, size_t count) = 0; + + // Read an integer. Throws Error on failure. + template T read_int(); + + // Read an integer into `value`. Throws Error on failure. + template void read_int(T& value); + + // Read a string of length `length`. Throws `core::Error` on failure. + std::string read_str(size_t length); +}; + +template +inline T +Reader::read_int() +{ + uint8_t buffer[sizeof(T)]; + const auto bytes_read = read(buffer, sizeof(T)); + if (bytes_read != sizeof(T)) { + throw core::Error("Read underflow"); + } + T value; + Util::big_endian_to_int(buffer, value); + return value; +} + +template +inline void +Reader::read_int(T& value) +{ + value = read_int(); +} + +inline std::string +Reader::read_str(const size_t length) +{ + std::string value(length, 0); + read(&value[0], length); + return value; +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/Sloppiness.hpp ccache-4.5.1/src/core/Sloppiness.hpp --- ccache-4.2.1/src/core/Sloppiness.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/Sloppiness.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,95 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace core { + +enum class Sloppy : uint32_t { + none = 0U, + + include_file_mtime = 1U << 0, + include_file_ctime = 1U << 1, + time_macros = 1U << 2, + pch_defines = 1U << 3, + // Allow us to match files based on their stats (size, mtime, ctime), without + // looking at their contents. + file_stat_matches = 1U << 4, + // Allow us to not include any system headers in the manifest include files, + // similar to -MM versus -M for dependencies. + system_headers = 1U << 5, + // Allow us to ignore ctimes when comparing file stats, so we can fake mtimes + // if we want to (it is much harder to fake ctimes, requires changing clock) + file_stat_matches_ctime = 1U << 6, + // Allow us to not include the -index-store-path option in the manifest hash. + clang_index_store = 1U << 7, + // Ignore locale settings. + locale = 1U << 8, + // Allow caching even if -fmodules is used. + modules = 1U << 9, + // Ignore virtual file system (VFS) overlay file. + ivfsoverlay = 1U << 10, +}; + +class Sloppiness +{ +public: + Sloppiness(Sloppy value = Sloppy::none); + explicit Sloppiness(uint32_t value); + + void enable(Sloppy value); + bool is_enabled(Sloppy value) const; + uint32_t to_bitmask() const; + +private: + Sloppy m_sloppiness = Sloppy::none; +}; + +// --- Inline implementations --- + +inline Sloppiness::Sloppiness(Sloppy value) : m_sloppiness(value) +{ +} + +inline Sloppiness::Sloppiness(uint32_t value) + : m_sloppiness(static_cast(value)) +{ +} + +inline void +Sloppiness::enable(Sloppy value) +{ + m_sloppiness = static_cast(static_cast(m_sloppiness) + | static_cast(value)); +} + +inline bool +Sloppiness::is_enabled(Sloppy value) const +{ + return static_cast(m_sloppiness) & static_cast(value); +} + +inline uint32_t +Sloppiness::to_bitmask() const +{ + return static_cast(m_sloppiness); +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/Statistic.hpp ccache-4.5.1/src/core/Statistic.hpp --- ccache-4.2.1/src/core/Statistic.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/Statistic.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,71 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +namespace core { + +// Statistics fields in storage order. +enum class Statistic { + none = 0, + compiler_produced_stdout = 1, + compile_failed = 2, + internal_error = 3, + cache_miss = 4, + preprocessor_error = 5, + could_not_find_compiler = 6, + missing_cache_file = 7, + preprocessed_cache_hit = 8, + bad_compiler_arguments = 9, + called_for_link = 10, + files_in_cache = 11, + cache_size_kibibyte = 12, + obsolete_max_files = 13, + obsolete_max_size = 14, + unsupported_source_language = 15, + bad_output_file = 16, + no_input_file = 17, + multiple_source_files = 18, + autoconf_test = 19, + unsupported_compiler_option = 20, + output_to_stdout = 21, + direct_cache_hit = 22, + compiler_produced_no_output = 23, + compiler_produced_empty_output = 24, + error_hashing_extra_file = 25, + compiler_check_failed = 26, + could_not_use_precompiled_header = 27, + called_for_preprocessing = 28, + cleanups_performed = 29, + unsupported_code_directive = 30, + stats_zeroed_timestamp = 31, + could_not_use_modules = 32, + direct_cache_miss = 33, + preprocessed_cache_miss = 34, + primary_storage_hit = 35, + primary_storage_miss = 36, + secondary_storage_hit = 37, + secondary_storage_miss = 38, + secondary_storage_error = 39, + secondary_storage_timeout = 40, + recache = 41, + + END +}; + +} // namespace core diff -Nru ccache-4.2.1/src/core/StatisticsCounters.cpp ccache-4.5.1/src/core/StatisticsCounters.cpp --- ccache-4.2.1/src/core/StatisticsCounters.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/StatisticsCounters.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,114 @@ +// Copyright (C) 2010-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatisticsCounters.hpp" + +#include + +#include + +namespace core { + +StatisticsCounters::StatisticsCounters() + : m_counters(static_cast(Statistic::END)) +{ +} + +StatisticsCounters::StatisticsCounters(const Statistic statistic) + : StatisticsCounters({statistic}) +{ +} + +StatisticsCounters::StatisticsCounters( + const std::initializer_list statistics) + : StatisticsCounters() +{ + for (auto st : statistics) { + increment(st); + } +} + +uint64_t +StatisticsCounters::get(Statistic statistic) const +{ + const auto index = static_cast(statistic); + ASSERT(index < static_cast(Statistic::END)); + return index < m_counters.size() ? m_counters[index] : 0; +} + +void +StatisticsCounters::set(Statistic statistic, uint64_t value) +{ + const auto index = static_cast(statistic); + ASSERT(index < static_cast(Statistic::END)); + m_counters[index] = value; +} + +uint64_t +StatisticsCounters::get_raw(size_t index) const +{ + ASSERT(index < size()); + return m_counters[index]; +} + +void +StatisticsCounters::set_raw(size_t index, uint64_t value) +{ + if (index >= m_counters.size()) { + m_counters.resize(index + 1); + } + m_counters[index] = value; +} + +void +StatisticsCounters::increment(Statistic statistic, int64_t value) +{ + const auto i = static_cast(statistic); + if (i >= m_counters.size()) { + m_counters.resize(i + 1); + } + auto& counter = m_counters[i]; + counter = + std::max(static_cast(0), static_cast(counter + value)); +} + +void +StatisticsCounters::increment(const StatisticsCounters& other) +{ + m_counters.resize(std::max(size(), other.size())); + for (size_t i = 0; i < other.size(); ++i) { + auto& counter = m_counters[i]; + counter = std::max(static_cast(0), + static_cast(counter + other.m_counters[i])); + } +} + +size_t +StatisticsCounters::size() const +{ + return m_counters.size(); +} + +bool +StatisticsCounters::all_zero() const +{ + return !std::any_of( + m_counters.begin(), m_counters.end(), [](unsigned v) { return v != 0; }); +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/StatisticsCounters.hpp ccache-4.5.1/src/core/StatisticsCounters.hpp --- ccache-4.2.1/src/core/StatisticsCounters.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/StatisticsCounters.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,57 @@ +// Copyright (C) 2010-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Statistic.hpp" + +#include +#include +#include +#include + +namespace core { + +// A simple wrapper around a vector of integers used for the statistics +// counters. +class StatisticsCounters +{ +public: + StatisticsCounters(); + StatisticsCounters(Statistic statistic); + StatisticsCounters(std::initializer_list statistics); + + uint64_t get(Statistic statistic) const; + void set(Statistic statistic, uint64_t value); + + uint64_t get_raw(size_t index) const; + void set_raw(size_t index, uint64_t value); + + void increment(Statistic statistic, int64_t value = 1); + void increment(const StatisticsCounters& other); + + size_t size() const; + + // Return true if all counters are zero, false otherwise. + bool all_zero() const; + +private: + std::vector m_counters; +}; + +} // namespace core diff -Nru ccache-4.2.1/src/core/Statistics.cpp ccache-4.5.1/src/core/Statistics.cpp --- ccache-4.2.1/src/core/Statistics.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/Statistics.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,387 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Statistics.hpp" + +#include +#include +#include +#include +#include +#include + +namespace core { + +using core::Statistic; + +const unsigned FLAG_NOZERO = 1U << 0; // don't zero with --zero-stats +const unsigned FLAG_NEVER = 1U << 1; // don't include in --print-stats +const unsigned FLAG_ERROR = 1U << 2; // include in error count +const unsigned FLAG_UNCACHEABLE = 1U << 3; // include in uncacheable count + +namespace { + +struct StatisticsField +{ + StatisticsField(const Statistic statistic_, + const char* const id_, + const char* const description_, + const unsigned flags_ = 0) + : statistic(statistic_), + id(id_), + description(description_), + flags(flags_) + { + } + + const Statistic statistic; + const char* const id; // for --print-stats + const char* const description; // for --show-stats --verbose + const unsigned flags; // bitmask of FLAG_* values +}; + +} // namespace + +#define FIELD(id, ...) \ + { \ + Statistic::id, #id, __VA_ARGS__ \ + } + +const StatisticsField k_statistics_fields[] = { + // Field "none" intentionally omitted. + FIELD(autoconf_test, "Autoconf compile/link", FLAG_UNCACHEABLE), + FIELD(bad_compiler_arguments, "Bad compiler arguments", FLAG_UNCACHEABLE), + FIELD(bad_output_file, "Could not write to output file", FLAG_ERROR), + FIELD(cache_miss, nullptr), + FIELD(cache_size_kibibyte, nullptr, FLAG_NOZERO), + FIELD(called_for_link, "Called for linking", FLAG_UNCACHEABLE), + FIELD(called_for_preprocessing, "Called for preprocessing", FLAG_UNCACHEABLE), + FIELD(cleanups_performed, nullptr), + FIELD(compile_failed, "Compilation failed", FLAG_UNCACHEABLE), + FIELD(compiler_check_failed, "Compiler check failed", FLAG_ERROR), + FIELD(compiler_produced_empty_output, + "Compiler produced empty output", + FLAG_UNCACHEABLE), + FIELD(compiler_produced_no_output, + "Compiler produced no output", + FLAG_UNCACHEABLE), + FIELD(compiler_produced_stdout, "Compiler produced stdout", FLAG_UNCACHEABLE), + FIELD(could_not_find_compiler, "Could not find compiler", FLAG_ERROR), + FIELD(could_not_use_modules, "Could not use modules", FLAG_UNCACHEABLE), + FIELD(could_not_use_precompiled_header, + "Could not use precompiled header", + FLAG_UNCACHEABLE), + FIELD(direct_cache_hit, nullptr), + FIELD(direct_cache_miss, nullptr), + FIELD(error_hashing_extra_file, "Error hashing extra file", FLAG_ERROR), + FIELD(files_in_cache, nullptr, FLAG_NOZERO), + FIELD(internal_error, "Internal error", FLAG_ERROR), + FIELD(missing_cache_file, "Missing cache file", FLAG_ERROR), + FIELD(multiple_source_files, "Multiple source files", FLAG_UNCACHEABLE), + FIELD(no_input_file, "No input file", FLAG_UNCACHEABLE), + FIELD(obsolete_max_files, nullptr, FLAG_NOZERO | FLAG_NEVER), + FIELD(obsolete_max_size, nullptr, FLAG_NOZERO | FLAG_NEVER), + FIELD(output_to_stdout, "Output to stdout", FLAG_UNCACHEABLE), + FIELD(preprocessed_cache_hit, nullptr), + FIELD(preprocessed_cache_miss, nullptr), + FIELD(preprocessor_error, "Preprocessing failed", FLAG_UNCACHEABLE), + FIELD(primary_storage_hit, nullptr), + FIELD(primary_storage_miss, nullptr), + FIELD(recache, "Forced recache", FLAG_UNCACHEABLE), + FIELD(secondary_storage_error, nullptr), + FIELD(secondary_storage_hit, nullptr), + FIELD(secondary_storage_miss, nullptr), + FIELD(secondary_storage_timeout, nullptr), + FIELD(stats_zeroed_timestamp, nullptr), + FIELD( + unsupported_code_directive, "Unsupported code directive", FLAG_UNCACHEABLE), + FIELD(unsupported_compiler_option, + "Unsupported compiler option", + FLAG_UNCACHEABLE), + FIELD(unsupported_source_language, + "Unsupported source language", + FLAG_UNCACHEABLE), +}; + +static_assert(sizeof(k_statistics_fields) / sizeof(k_statistics_fields[0]) + == static_cast(Statistic::END) - 1, + "incorrect number of fields"); + +static std::string +format_timestamp(const uint64_t value) +{ + if (value == 0) { + return "never"; + } else { + const auto tm = Util::localtime(value); + char buffer[100] = "?"; + if (tm) { + strftime(buffer, sizeof(buffer), "%c", &*tm); + } + return buffer; + } +} + +static std::string +percent(const uint64_t nominator, const uint64_t denominator) +{ + if (denominator == 0) { + return ""; + } else if (nominator >= denominator) { + return FMT("({:.1f} %)", (100.0 * nominator) / denominator); + } else { + return FMT("({:.2f} %)", (100.0 * nominator) / denominator); + } +} + +Statistics::Statistics(const StatisticsCounters& counters) + : m_counters(counters) +{ +} + +std::vector +Statistics::get_statistics_ids() const +{ + std::vector result; + for (const auto& field : k_statistics_fields) { + if (m_counters.get(field.statistic) != 0 && !(field.flags & FLAG_NOZERO)) { + result.emplace_back(field.id); + } + } + std::sort(result.begin(), result.end()); + return result; +} + +uint64_t +Statistics::count_stats(const unsigned flags) const +{ + uint64_t sum = 0; + for (const auto& field : k_statistics_fields) { + if (field.flags & flags) { + sum += m_counters.get(field.statistic); + } + } + return sum; +} + +std::vector> +Statistics::get_stats(unsigned flags, const bool all) const +{ + std::vector> result; + for (const auto& field : k_statistics_fields) { + const auto count = m_counters.get(field.statistic); + if ((field.flags & flags) && (all || count > 0)) { + result.emplace_back(field.description, count); + } + } + return result; +} + +std::string +Statistics::format_human_readable(const Config& config, + const time_t last_updated, + const uint8_t verbosity, + const bool from_log) const +{ + util::TextTable table; + using C = util::TextTable::Cell; + +#define S(x_) m_counters.get(Statistic::x_) + + const uint64_t d_hits = S(direct_cache_hit); + const uint64_t d_misses = S(direct_cache_miss); + const uint64_t p_hits = S(preprocessed_cache_hit); + const uint64_t p_misses = S(preprocessed_cache_miss); + const uint64_t hits = d_hits + p_hits; + const uint64_t misses = S(cache_miss); + + table.add_heading("Summary:"); + if (verbosity > 0 && !from_log) { + table.add_row({" Cache directory:", C(config.cache_dir()).colspan(4)}); + table.add_row( + {" Primary config:", C(config.primary_config_path()).colspan(4)}); + table.add_row( + {" Secondary config:", C(config.secondary_config_path()).colspan(4)}); + table.add_row( + {" Stats updated:", C(format_timestamp(last_updated)).colspan(4)}); + if (verbosity > 1) { + const uint64_t last_zeroed = S(stats_zeroed_timestamp); + table.add_row( + {" Stats zeroed:", C(format_timestamp(last_zeroed)).colspan(4)}); + } + } + table.add_row({ + " Hits:", + hits, + "/", + hits + misses, + percent(hits, hits + misses), + }); + table.add_row({ + " Direct:", + d_hits, + "/", + d_hits + d_misses, + percent(d_hits, d_hits + d_misses), + }); + table.add_row({ + " Preprocessed:", + p_hits, + "/", + p_hits + p_misses, + percent(p_hits, p_hits + p_misses), + }); + table.add_row({" Misses:", misses}); + table.add_row({" Direct:", d_misses}); + table.add_row({" Preprocessed:", p_misses}); + + const auto errors = count_stats(FLAG_ERROR); + const auto uncacheable = count_stats(FLAG_UNCACHEABLE); + if (verbosity > 1 || errors > 0) { + table.add_row({" Errors:", errors}); + } + if (verbosity > 1 || uncacheable > 0) { + table.add_row({" Uncacheable:", uncacheable}); + } + + const uint64_t g = 1'000'000'000; + const uint64_t pri_hits = S(primary_storage_hit); + const uint64_t pri_misses = S(primary_storage_miss); + const uint64_t pri_size = S(cache_size_kibibyte) * 1024; + const uint64_t cleanups = S(cleanups_performed); + table.add_heading("Primary storage:"); + table.add_row({ + " Hits:", + pri_hits, + "/", + pri_hits + pri_misses, + percent(pri_hits, pri_hits + pri_misses), + }); + table.add_row({" Misses:", pri_misses}); + if (!from_log) { + table.add_row({ + " Cache size (GB):", + C(FMT("{:.2f}", static_cast(pri_size) / g)).right_align(), + "/", + C(FMT("{:.2f}", static_cast(config.max_size()) / g)) + .right_align(), + percent(pri_size, config.max_size()), + }); + if (verbosity > 0) { + std::vector cells{" Files:", S(files_in_cache)}; + if (config.max_files() > 0) { + cells.emplace_back("/"); + cells.emplace_back(config.max_files()); + cells.emplace_back(percent(S(files_in_cache), config.max_files())); + } + table.add_row(cells); + } + if (cleanups > 0) { + table.add_row({" Cleanups:", cleanups}); + } + } + + const uint64_t sec_hits = S(secondary_storage_hit); + const uint64_t sec_misses = S(secondary_storage_miss); + const uint64_t sec_errors = S(secondary_storage_error); + const uint64_t sec_timeouts = S(secondary_storage_timeout); + + if (verbosity > 1 || sec_hits + sec_misses + sec_errors + sec_timeouts > 0) { + table.add_heading("Secondary storage:"); + table.add_row({ + " Hits:", + sec_hits, + "/", + sec_hits + sec_misses, + percent(sec_hits, sec_hits + pri_misses), + }); + table.add_row({" Misses:", sec_misses}); + if (verbosity > 1 || sec_errors > 0) { + table.add_row({" Errors:", sec_errors}); + } + if (verbosity > 1 || sec_timeouts > 0) { + table.add_row({" Timeouts:", sec_timeouts}); + } + } + + auto cmp_fn = [](const auto& e1, const auto& e2) { + return e1.first.compare(e2.first) < 0; + }; + + if (verbosity > 1 || (verbosity == 1 && errors > 0)) { + auto error_stats = get_stats(FLAG_ERROR, verbosity > 1); + std::sort(error_stats.begin(), error_stats.end(), cmp_fn); + table.add_heading("Errors:"); + for (const auto& descr_count : error_stats) { + table.add_row({FMT(" {}:", descr_count.first), descr_count.second}); + } + } + + if (verbosity > 1 || (verbosity == 1 && uncacheable > 0)) { + auto uncacheable_stats = get_stats(FLAG_UNCACHEABLE, verbosity > 1); + std::sort(uncacheable_stats.begin(), uncacheable_stats.end(), cmp_fn); + table.add_heading("Uncacheable:"); + for (const auto& descr_count : uncacheable_stats) { + table.add_row({FMT(" {}:", descr_count.first), descr_count.second}); + } + } + + return table.render(); +} + +std::string +Statistics::format_machine_readable(const time_t last_updated) const +{ + std::vector lines; + + lines.push_back(FMT("stats_updated_timestamp\t{}\n", last_updated)); + + for (const auto& field : k_statistics_fields) { + if (!(field.flags & FLAG_NEVER)) { + lines.push_back( + FMT("{}\t{}\n", field.id, m_counters.get(field.statistic))); + } + } + + std::sort(lines.begin(), lines.end()); + return util::join(lines, ""); +} + +std::unordered_map +Statistics::get_id_map() +{ + std::unordered_map result; + for (const auto& field : k_statistics_fields) { + result[field.id] = field.statistic; + } + return result; +} + +std::vector +Statistics::get_zeroable_fields() +{ + std::vector result; + for (const auto& field : k_statistics_fields) { + if (!(field.flags & FLAG_NOZERO)) { + result.push_back(field.statistic); + } + } + return result; +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/Statistics.hpp ccache-4.5.1/src/core/Statistics.hpp --- ccache-4.2.1/src/core/Statistics.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/Statistics.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,70 @@ +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include +#include + +class Config; + +namespace core { + +class Statistics +{ +public: + Statistics(const StatisticsCounters& counters); + + // Return machine-readable strings representing the statistics counters. + std::vector get_statistics_ids() const; + + // Format cache statistics in human-readable format. + std::string format_human_readable(const Config& config, + time_t last_updated, + uint8_t verbosity, + bool from_log) const; + + // Format cache statistics in machine-readable format. + std::string format_machine_readable(time_t last_updated) const; + + const StatisticsCounters& counters() const; + + static std::unordered_map get_id_map(); + + static std::vector get_zeroable_fields(); + +private: + const StatisticsCounters m_counters; + + uint64_t count_stats(unsigned flags) const; + std::vector> get_stats(unsigned flags, + bool all) const; +}; + +// --- Inline implementations --- + +inline const StatisticsCounters& +Statistics::counters() const +{ + return m_counters; +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/StatsLog.cpp ccache-4.5.1/src/core/StatsLog.cpp --- ccache-4.2.1/src/core/StatsLog.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/StatsLog.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,71 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatsLog.hpp" + +#include +#include +#include +#include + +#include + +namespace core { + +StatisticsCounters +StatsLog::read() const +{ + core::StatisticsCounters counters; + + const auto id_map = Statistics::get_id_map(); + + std::ifstream in(m_path); + std::string line; + while (std::getline(in, line, '\n')) { + if (line[0] == '#') { + continue; + } + const auto entry = id_map.find(line); + if (entry != id_map.end()) { + Statistic statistic = entry->second; + counters.increment(statistic, 1); + } else { + LOG("Unknown statistic: {}", line); + } + } + + return counters; +} + +void +StatsLog::log_result(const std::string& input_file, + const std::vector& result_ids) +{ + File file(m_path, "ab"); + if (!file) { + LOG("Failed to open {}: {}", m_path, strerror(errno)); + return; + } + + PRINT(*file, "# {}\n", input_file); + for (const auto& id : result_ids) { + PRINT(*file, "{}\n", id); + } +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/StatsLog.hpp ccache-4.5.1/src/core/StatsLog.hpp --- ccache-4.2.1/src/core/StatsLog.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/StatsLog.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,47 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "StatisticsCounters.hpp" + +#include +#include + +namespace core { + +class StatsLog +{ +public: + StatsLog(const std::string& path); + + StatisticsCounters read() const; + void log_result(const std::string& input_file, + const std::vector& result_ids); + +private: + const std::string m_path; +}; + +// --- Inline implementations --- + +inline StatsLog::StatsLog(const std::string& path) : m_path(path) +{ +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/types.cpp ccache-4.5.1/src/core/types.cpp --- ccache-4.2.1/src/core/types.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/types.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,38 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "types.hpp" + +namespace core { + +std::string +to_string(const CacheEntryType type) +{ + switch (type) { + case CacheEntryType::manifest: + return "manifest"; + + case CacheEntryType::result: + return "result"; + + default: + return "unknown"; + } +} + +} // namespace core diff -Nru ccache-4.2.1/src/core/types.hpp ccache-4.5.1/src/core/types.hpp --- ccache-4.2.1/src/core/types.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/types.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,30 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace core { + +enum class CacheEntryType : uint8_t { result = 0, manifest = 1 }; + +std::string to_string(CacheEntryType type); + +} // namespace core diff -Nru ccache-4.2.1/src/core/wincompat.hpp ccache-4.5.1/src/core/wincompat.hpp --- ccache-4.2.1/src/core/wincompat.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/wincompat.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,109 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#ifdef _WIN32 +# include + +# define NOMINMAX 1 +# define STDIN_FILENO 0 +# define STDOUT_FILENO 1 +# define STDERR_FILENO 2 + +# ifdef _MSC_VER +# define PATH_MAX MAX_PATH +# endif // _MSC_VER + +// From: +// http://mesos.apache.org/api/latest/c++/3rdparty_2stout_2include_2stout_2windows_8hpp_source.html +# ifdef _MSC_VER +const mode_t S_IRUSR = mode_t(_S_IREAD); +const mode_t S_IWUSR = mode_t(_S_IWRITE); +# endif + +# ifndef S_IFIFO +# define S_IFIFO 0x1000 +# endif + +# ifndef S_IFBLK +# define S_IFBLK 0x6000 +# endif + +# ifndef S_IFLNK +# define S_IFLNK 0xA000 +# endif + +# ifndef S_ISREG +# define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) +# endif +# ifndef S_ISDIR +# define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) +# endif +# ifndef S_ISFIFO +# define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO) +# endif +# ifndef S_ISCHR +# define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR) +# endif +# ifndef S_ISLNK +# define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK) +# endif +# ifndef S_ISBLK +# define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK) +# endif + +# include +# include +# include +# include +# define NOMINMAX 1 +# define WIN32_NO_STATUS +// clang-format off +# include +# include // NTSTATUS +# include // struct timeval +// clang-format on +# undef WIN32_NO_STATUS +# include +# define mkdir(a, b) _mkdir(a) + +// Protect against incidental use of MinGW execv. +# define execv(a, b) do_not_call_execv_on_windows + +# ifdef _MSC_VER +# define PATH_MAX MAX_PATH +# endif + +# ifdef _MSC_VER +# define DLLIMPORT __declspec(dllimport) +# else +# define DLLIMPORT +# endif + +# define STDIN_FILENO 0 +# define STDOUT_FILENO 1 +# define STDERR_FILENO 2 + +# ifndef O_BINARY +# define O_BINARY 0 +# endif + +#else +# define DLLIMPORT +#endif // _WIN32 diff -Nru ccache-4.2.1/src/core/Writer.hpp ccache-4.5.1/src/core/Writer.hpp --- ccache-4.2.1/src/core/Writer.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/core/Writer.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,64 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include +#include +#include + +namespace core { + +class Writer +{ +public: + virtual ~Writer() = default; + + // Write `count` bytes from `data`. Throws `core::Error` on failure. + virtual void write(const void* data, size_t count) = 0; + + // Write integer `value`. Throws `core::Error` on failure. + template void write_int(T value); + + // Write `value`. Throws `core::Error` on failure. + void write_str(const std::string& value); + + // Finalize writing, e.g. flush written bytes and potentially check for error + // states. Throws `core::Error` on failure. + virtual void finalize() = 0; +}; + +template +inline void +Writer::write_int(const T value) +{ + uint8_t buffer[sizeof(T)]; + Util::int_to_big_endian(value, buffer); + write(buffer, sizeof(T)); +} + +inline void +Writer::write_str(const std::string& value) +{ + write(value.data(), value.length()); +} + +} // namespace core diff -Nru ccache-4.2.1/src/Counters.cpp ccache-4.5.1/src/Counters.cpp --- ccache-4.2.1/src/Counters.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Counters.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,96 +0,0 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Counters.hpp" - -#include "Statistic.hpp" -#include "assertions.hpp" - -#include - -Counters::Counters() : m_counters(static_cast(Statistic::END)) -{ -} - -uint64_t -Counters::get(Statistic statistic) const -{ - const auto index = static_cast(statistic); - ASSERT(index < static_cast(Statistic::END)); - return index < m_counters.size() ? m_counters[index] : 0; -} - -void -Counters::set(Statistic statistic, uint64_t value) -{ - const auto index = static_cast(statistic); - ASSERT(index < static_cast(Statistic::END)); - m_counters[index] = value; -} - -uint64_t -Counters::get_raw(size_t index) const -{ - ASSERT(index < size()); - return m_counters[index]; -} - -void -Counters::set_raw(size_t index, uint64_t value) -{ - if (index >= m_counters.size()) { - m_counters.resize(index + 1); - } - m_counters[index] = value; -} - -void -Counters::increment(Statistic statistic, int64_t value) -{ - const auto i = static_cast(statistic); - if (i >= m_counters.size()) { - m_counters.resize(i + 1); - } - auto& counter = m_counters[i]; - counter = - std::max(static_cast(0), static_cast(counter + value)); -} - -void -Counters::increment(const Counters& other) -{ - m_counters.resize(std::max(size(), other.size())); - for (size_t i = 0; i < other.size(); ++i) { - auto& counter = m_counters[i]; - counter = std::max(static_cast(0), - static_cast(counter + other.m_counters[i])); - } -} - -size_t -Counters::size() const -{ - return m_counters.size(); -} - -bool -Counters::all_zero() const -{ - return !std::any_of( - m_counters.begin(), m_counters.end(), [](unsigned v) { return v != 0; }); -} diff -Nru ccache-4.2.1/src/Counters.hpp ccache-4.5.1/src/Counters.hpp --- ccache-4.2.1/src/Counters.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Counters.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include - -enum class Statistic; - -// A simple wrapper around a vector of integers used for the statistics -// counters. -class Counters -{ -public: - Counters(); - - uint64_t get(Statistic statistic) const; - void set(Statistic statistic, uint64_t value); - - uint64_t get_raw(size_t index) const; - void set_raw(size_t index, uint64_t value); - - void increment(Statistic statistic, int64_t value = 1); - void increment(const Counters& other); - - size_t size() const; - - // Return true if all counters are zero, false otherwise. - bool all_zero() const; - -private: - std::vector m_counters; -}; diff -Nru ccache-4.2.1/src/Decompressor.cpp ccache-4.5.1/src/Decompressor.cpp --- ccache-4.2.1/src/Decompressor.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Decompressor.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,38 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Decompressor.hpp" - -#include "NullDecompressor.hpp" -#include "StdMakeUnique.hpp" -#include "ZstdDecompressor.hpp" -#include "assertions.hpp" - -std::unique_ptr -Decompressor::create_from_type(Compression::Type type, FILE* stream) -{ - switch (type) { - case Compression::Type::none: - return std::make_unique(stream); - - case Compression::Type::zstd: - return std::make_unique(stream); - } - - ASSERT(false); -} diff -Nru ccache-4.2.1/src/Decompressor.hpp ccache-4.5.1/src/Decompressor.hpp --- ccache-4.2.1/src/Decompressor.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Decompressor.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,54 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Compression.hpp" - -#include - -class Decompressor -{ -public: - virtual ~Decompressor() = default; - - // Create a decompressor for the specified type. - // - // Parameters: - // - type: The type. - // - stream: The stream to read from. - static std::unique_ptr create_from_type(Compression::Type type, - FILE* stream); - - // Read data into a buffer from the compressed stream. - // - // Parameters: - // - data: Buffer to write decompressed data to. - // - count: How many bytes to write. - // - // Throws Error on failure. - virtual void read(void* data, size_t count) = 0; - - // Finalize decompression. - // - // This method checks that the end state of the compressed stream is correct - // and throws Error if not. - virtual void finalize() = 0; -}; diff -Nru ccache-4.2.1/src/Depfile.cpp ccache-4.5.1/src/Depfile.cpp --- ccache-4.2.1/src/Depfile.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Depfile.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -23,6 +23,9 @@ #include "Logging.hpp" #include "assertions.hpp" +#include +#include + static inline bool is_blank(const std::string& s) { @@ -69,17 +72,18 @@ adjusted_file_content.reserve(file_content.size()); bool content_rewritten = false; - for (const auto& line : Util::split_into_views(file_content, "\n")) { + for (const auto line : util::Tokenizer( + file_content, "\n", util::Tokenizer::Mode::skip_last_empty)) { const auto tokens = Util::split_into_views(line, " \t"); for (size_t i = 0; i < tokens.size(); ++i) { - DEBUG_ASSERT(line.length() > 0); // line.empty() -> no tokens + DEBUG_ASSERT(!line.empty()); // line.empty() -> no tokens if (i > 0 || line[0] == ' ' || line[0] == '\t') { adjusted_file_content.push_back(' '); } const auto& token = tokens[i]; bool token_rewritten = false; - if (Util::is_absolute_path(token)) { + if (util::is_absolute_path(token)) { const auto new_path = Util::make_relative_path(ctx, token); if (new_path != token) { adjusted_file_content.append(new_path); @@ -120,7 +124,7 @@ std::string file_content; try { file_content = Util::read_file(output_dep); - } catch (const Error& e) { + } catch (const core::Error& e) { LOG("Cannot open dependency file {}: {}", output_dep, e.what()); return; } diff -Nru ccache-4.2.1/src/Depfile.hpp ccache-4.5.1/src/Depfile.hpp --- ccache-4.2.1/src/Depfile.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Depfile.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // diff -Nru ccache-4.2.1/src/Digest.hpp ccache-4.5.1/src/Digest.hpp --- ccache-4.2.1/src/Digest.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Digest.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,12 +18,11 @@ #pragma once -#include "system.hpp" - #include "Util.hpp" #include "third_party/fmt/core.h" +#include #include // Digest represents the binary form of the final digest (AKA hash or checksum) diff -Nru ccache-4.2.1/src/exceptions.hpp ccache-4.5.1/src/exceptions.hpp --- ccache-4.2.1/src/exceptions.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/exceptions.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,81 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "FormatNonstdStringView.hpp" - -#include "third_party/fmt/core.h" -#include "third_party/nonstd/optional.hpp" - -#include -#include -#include - -// Don't throw or catch ErrorBase directly, use a subclass. -class ErrorBase : public std::runtime_error -{ - using std::runtime_error::runtime_error; -}; - -// Throw an Error to indicate a potentially non-fatal error that may be caught -// and handled by callers. An uncaught Error that reaches the top level will be -// treated similar to Fatal. -class Error : public ErrorBase -{ -public: - // Special case: If given only one string, don't parse it as a format string. - Error(const std::string& message); - - // `args` are forwarded to `fmt::format`. - template inline Error(T&&... args); -}; - -inline Error::Error(const std::string& message) : ErrorBase(message) -{ -} - -template -inline Error::Error(T&&... args) - : ErrorBase(fmt::format(std::forward(args)...)) -{ -} - -// Throw a Fatal to make ccache print the error message to stderr and exit -// with a non-zero exit code. -class Fatal : public ErrorBase -{ -public: - // Special case: If given only one string, don't parse it as a format string. - Fatal(const std::string& message); - - // `args` are forwarded to `fmt::format`. - template inline Fatal(T&&... args); -}; - -inline Fatal::Fatal(const std::string& message) : ErrorBase(message) -{ -} - -template -inline Fatal::Fatal(T&&... args) - : ErrorBase(fmt::format(std::forward(args)...)) -{ -} diff -Nru ccache-4.2.1/src/execute.cpp ccache-4.5.1/src/execute.cpp --- ccache-4.2.1/src/execute.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/execute.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -27,11 +27,23 @@ #include "Stat.hpp" #include "TemporaryFile.hpp" #include "Util.hpp" +#include "Win32Util.hpp" #include "fmtmacros.hpp" +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +#ifdef HAVE_SYS_WAIT_H +# include +#endif + #ifdef _WIN32 # include "Finalizer.hpp" -# include "Win32Util.hpp" #endif using nonstd::string_view; @@ -192,7 +204,7 @@ } if (ctx.compiler_pid == -1) { - throw Fatal("Failed to fork: {}", strerror(errno)); + throw core::Fatal("Failed to fork: {}", strerror(errno)); } if (ctx.compiler_pid == 0) { @@ -214,7 +226,7 @@ if (result == -1 && errno == EINTR) { continue; } - throw Fatal("waitpid failed: {}", strerror(errno)); + throw core::Fatal("waitpid failed: {}", strerror(errno)); } { @@ -241,7 +253,7 @@ const std::string& name, const std::string& exclude_name) { - if (Util::is_absolute_path(name)) { + if (util::is_absolute_path(name)) { return name; } @@ -268,7 +280,7 @@ // Search the path looking for the first compiler of the right name that isn't // us. - for (const std::string& dir : Util::split_into_strings(path, PATH_DELIM)) { + for (const std::string& dir : util::split_path_list(path)) { #ifdef _WIN32 char namebuf[MAX_PATH]; int ret = SearchPath( diff -Nru ccache-4.2.1/src/execute.hpp ccache-4.5.1/src/execute.hpp --- ccache-4.2.1/src/execute.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/execute.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "Fd.hpp" #include diff -Nru ccache-4.2.1/src/Fd.cpp ccache-4.5.1/src/Fd.cpp --- ccache-4.2.1/src/Fd.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/Fd.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,31 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Fd.hpp" + +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +bool +Fd::close() +{ + return m_fd != -1 && ::close(release()) == 0; +} diff -Nru ccache-4.2.1/src/Fd.hpp ccache-4.5.1/src/Fd.hpp --- ccache-4.2.1/src/Fd.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Fd.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" #include "assertions.hpp" @@ -72,10 +70,8 @@ return m_fd; } -// clang-format off inline int Fd::operator*() const -// clang-format on { ASSERT(m_fd != -1); return m_fd; @@ -89,12 +85,6 @@ return *this; } -inline bool -Fd::close() -{ - return m_fd != -1 && ::close(release()) == 0; -} - inline int Fd::release() { diff -Nru ccache-4.2.1/src/File.hpp ccache-4.5.1/src/File.hpp --- ccache-4.2.1/src/File.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/File.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,10 +18,9 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" +#include #include class File : public NonCopyable @@ -89,10 +88,8 @@ return m_file; } -// clang-format off inline FILE* File::operator*() const -// clang-format on { return m_file; } diff -Nru ccache-4.2.1/src/Finalizer.hpp ccache-4.5.1/src/Finalizer.hpp --- ccache-4.2.1/src/Finalizer.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Finalizer.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include class Finalizer diff -Nru ccache-4.2.1/src/fmtmacros.hpp ccache-4.5.1/src/fmtmacros.hpp --- ccache-4.2.1/src/fmtmacros.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/fmtmacros.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,10 @@ #pragma once -#include "third_party/fmt/core.h" -#include "third_party/fmt/format.h" +#include + +#include +#include // Convenience macro for calling `fmt::format` with `FMT_STRING` around the // format string literal. diff -Nru ccache-4.2.1/src/FormatNonstdStringView.hpp ccache-4.5.1/src/FormatNonstdStringView.hpp --- ccache-4.2.1/src/FormatNonstdStringView.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/FormatNonstdStringView.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "third_party/fmt/core.h" #include "third_party/nonstd/string_view.hpp" diff -Nru ccache-4.2.1/src/Hash.cpp ccache-4.5.1/src/Hash.cpp --- ccache-4.2.1/src/Hash.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Hash.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -22,6 +22,16 @@ #include "Logging.hpp" #include "fmtmacros.hpp" +#include + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + using nonstd::string_view; const string_view HASH_DELIMITER("\000cCaChE\000", 8); diff -Nru ccache-4.2.1/src/Hash.hpp ccache-4.5.1/src/Hash.hpp --- ccache-4.2.1/src/Hash.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Hash.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,13 +18,14 @@ #pragma once -#include "system.hpp" - #include "Digest.hpp" #include "third_party/blake3/blake3.h" #include "third_party/nonstd/string_view.hpp" +#include +#include + // This class represents a hash state. class Hash { diff -Nru ccache-4.2.1/src/hashutil.cpp ccache-4.5.1/src/hashutil.cpp --- ccache-4.2.1/src/hashutil.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/hashutil.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -23,20 +23,29 @@ #include "Context.hpp" #include "Hash.hpp" #include "Logging.hpp" -#include "Sloppiness.hpp" #include "Stat.hpp" +#include "Util.hpp" +#include "Win32Util.hpp" #include "execute.hpp" #include "fmtmacros.hpp" #include "macroskip.hpp" -#include "third_party/blake3/blake3_cpu_supports_avx2.h" +#include +#include +#include #ifdef INODE_CACHE_SUPPORTED # include "InodeCache.hpp" #endif -#ifdef _WIN32 -# include "Win32Util.hpp" +#include "third_party/blake3/blake3_cpu_supports_avx2.h" + +#ifdef HAVE_UNISTD_H +# include +#endif + +#ifdef HAVE_SYS_WAIT_H +# include #endif #ifdef HAVE_AVX2 @@ -180,7 +189,7 @@ std::string data; try { data = Util::read_file(path, size_hint); - } catch (Error&) { + } catch (core::Error&) { return HASH_SOURCE_CODE_ERROR; } int result = hash_source_code_string(ctx, hash, data, path); @@ -195,7 +204,7 @@ if (Util::is_precompiled_header(path)) { return InodeCache::ContentType::precompiled_header; } - if (config.sloppiness() & SLOPPY_TIME_MACROS) { + if (config.sloppiness().is_enabled(core::Sloppy::time_macros)) { return InodeCache::ContentType::code_with_sloppy_time_macros; } return InodeCache::ContentType::code; @@ -225,7 +234,7 @@ // Check for __DATE__, __TIME__ and __TIMESTAMP__if the sloppiness // configuration tells us we should. - if (!(ctx.config.sloppiness() & SLOPPY_TIME_MACROS)) { + if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::time_macros))) { result |= check_for_temporal_macros(str); } @@ -367,14 +376,14 @@ const std::string& compiler) { #ifdef _WIN32 - std::string adjusted_command = Util::strip_whitespace(command); + std::string adjusted_command = util::strip_whitespace(command); // Add "echo" command. bool using_cmd_exe; - if (Util::starts_with(adjusted_command, "echo")) { + if (util::starts_with(adjusted_command, "echo")) { adjusted_command = FMT("cmd.exe /c \"{}\"", adjusted_command); using_cmd_exe = true; - } else if (Util::starts_with(adjusted_command, "%compiler%") + } else if (util::starts_with(adjusted_command, "%compiler%") && compiler == "echo") { adjusted_command = FMT("cmd.exe /c \"{}{}\"", compiler, adjusted_command.substr(10)); @@ -462,12 +471,12 @@ #else int pipefd[2]; if (pipe(pipefd) == -1) { - throw Fatal("pipe failed: {}", strerror(errno)); + throw core::Fatal("pipe failed: {}", strerror(errno)); } pid_t pid = fork(); if (pid == -1) { - throw Fatal("fork failed: {}", strerror(errno)); + throw core::Fatal("fork failed: {}", strerror(errno)); } if (pid == 0) { diff -Nru ccache-4.2.1/src/hashutil.hpp ccache-4.5.1/src/hashutil.hpp --- ccache-4.2.1/src/hashutil.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/hashutil.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors +// Copyright (C) 2009-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,10 +18,9 @@ #pragma once -#include "system.hpp" - #include "third_party/nonstd/string_view.hpp" +#include #include class Config; diff -Nru ccache-4.2.1/src/InodeCache.cpp ccache-4.5.1/src/InodeCache.cpp --- ccache-4.2.1/src/InodeCache.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/InodeCache.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -19,6 +19,7 @@ #include "InodeCache.hpp" #include "Config.hpp" +#include "Digest.hpp" #include "Fd.hpp" #include "Finalizer.hpp" #include "Hash.hpp" @@ -28,9 +29,12 @@ #include "Util.hpp" #include "fmtmacros.hpp" -#include +#include #include #include +#include + +#include #include // The inode cache resides on a file that is mapped into shared memory by @@ -62,7 +66,7 @@ static_assert(Digest::size() == 20, "Increment version number if size of digest is changed."); -static_assert(IS_TRIVIALLY_COPYABLE(Digest), +static_assert(std::is_trivially_copyable::value, "Digest is expected to be trivially copyable."); static_assert( @@ -369,7 +373,7 @@ } bool found = false; - const bool success = with_bucket(key_digest, [&](Bucket* const bucket) { + const bool success = with_bucket(key_digest, [&](const auto bucket) { for (uint32_t i = 0; i < k_num_entries; ++i) { if (bucket->entries[i].key_digest == key_digest) { if (i > 0) { @@ -422,7 +426,7 @@ return false; } - const bool success = with_bucket(key_digest, [&](Bucket* const bucket) { + const bool success = with_bucket(key_digest, [&](const auto bucket) { memmove(&bucket->entries[1], &bucket->entries[0], sizeof(Entry) * (k_num_entries - 1)); diff -Nru ccache-4.2.1/src/InodeCache.hpp ccache-4.5.1/src/InodeCache.hpp --- ccache-4.2.1/src/InodeCache.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/InodeCache.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,10 +18,7 @@ #pragma once -#include "system.hpp" - -#include "config.h" - +#include #include #include diff -Nru ccache-4.2.1/src/language.hpp ccache-4.5.1/src/language.hpp --- ccache-4.2.1/src/language.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/language.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "Config.hpp" #include diff -Nru ccache-4.2.1/src/Lockfile.cpp ccache-4.5.1/src/Lockfile.cpp --- ccache-4.2.1/src/Lockfile.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Lockfile.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,14 +20,19 @@ #include "Logging.hpp" #include "Util.hpp" +#include "Win32Util.hpp" #include "fmtmacros.hpp" -#ifdef _WIN32 -# include "Win32Util.hpp" -#endif +#include #include "third_party/fmt/core.h" +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #include #include #include @@ -109,7 +114,7 @@ LOG("lockfile_acquire: failed to acquire {}; sleeping {} microseconds", lockfile, to_sleep); - usleep(to_sleep); + std::this_thread::sleep_for(std::chrono::microseconds(to_sleep)); slept += to_sleep; to_sleep = std::min(max_to_sleep, 2 * to_sleep); } else if (content != initial_content) { @@ -134,9 +139,9 @@ HANDLE do_acquire_win32(const std::string& lockfile, uint32_t staleness_limit) { - unsigned to_sleep = 1000; // Microseconds. - unsigned max_to_sleep = 10000; // Microseconds. - unsigned slept = 0; // Microseconds. + const uint32_t max_to_sleep = 10000; // Microseconds. + uint32_t to_sleep = 1000; // Microseconds. + uint32_t slept = 0; // Microseconds. HANDLE handle; while (true) { @@ -154,10 +159,6 @@ } DWORD error = GetLastError(); - LOG("lockfile_acquire: CreateFile {}: {} ({})", - lockfile, - Win32Util::error_message(error), - error); if (error == ERROR_PATH_NOT_FOUND) { // Directory doesn't exist? if (Util::create_dir(Util::dir_name(lockfile))) { @@ -166,6 +167,11 @@ } } + LOG("lockfile_acquire: CreateFile {}: {} ({})", + lockfile, + Win32Util::error_message(error), + error); + // ERROR_SHARING_VIOLATION: lock already held. // ERROR_ACCESS_DENIED: maybe pending delete. if (error != ERROR_SHARING_VIOLATION && error != ERROR_ACCESS_DENIED) { @@ -181,7 +187,7 @@ LOG("lockfile_acquire: failed to acquire {}; sleeping {} microseconds", lockfile, to_sleep); - usleep(to_sleep); + std::this_thread::sleep_for(std::chrono::microseconds(to_sleep)); slept += to_sleep; to_sleep = std::min(max_to_sleep, 2 * to_sleep); } @@ -221,3 +227,13 @@ #endif } } + +bool +Lockfile::acquired() const +{ +#ifndef _WIN32 + return m_acquired; +#else + return m_handle != INVALID_HANDLE_VALUE; +#endif +} diff -Nru ccache-4.2.1/src/Lockfile.hpp ccache-4.5.1/src/Lockfile.hpp --- ccache-4.2.1/src/Lockfile.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Lockfile.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,7 @@ #pragma once -#include "system.hpp" - +#include #include class Lockfile @@ -40,16 +39,6 @@ #ifndef _WIN32 bool m_acquired = false; #else - HANDLE m_handle = nullptr; + void* m_handle = nullptr; #endif }; - -inline bool -Lockfile::acquired() const -{ -#ifndef _WIN32 - return m_acquired; -#else - return m_handle != INVALID_HANDLE_VALUE; -#endif -} diff -Nru ccache-4.2.1/src/Logging.cpp ccache-4.5.1/src/Logging.cpp --- ccache-4.2.1/src/Logging.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Logging.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -22,10 +22,16 @@ #include "Config.hpp" #include "File.hpp" #include "Util.hpp" -#include "exceptions.hpp" +#include "Win32Util.hpp" #include "execute.hpp" #include "fmtmacros.hpp" +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #ifdef HAVE_SYSLOG_H # include #endif @@ -39,12 +45,6 @@ # endif #endif -#ifdef _WIN32 -# include -# include -# include -#endif - using nonstd::string_view; namespace { @@ -54,7 +54,7 @@ File logfile; // Whether to use syslog() instead. -bool use_syslog; +bool use_syslog = false; // Buffer used for logs in debug mode. std::string debug_log_buffer; diff -Nru ccache-4.2.1/src/Logging.hpp ccache-4.5.1/src/Logging.hpp --- ccache-4.2.1/src/Logging.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Logging.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "FormatNonstdStringView.hpp" #include "third_party/fmt/core.h" @@ -39,13 +37,12 @@ } while (false) // Log a message (plus a newline character) described by a format string with at -// least one placeholder. `format` is compile-time checked if CMAKE_CXX_STANDARD -// >= 14. +// least one placeholder. `format` is checked at compile time. #define LOG(format_, ...) LOG_RAW(fmt::format(FMT_STRING(format_), __VA_ARGS__)) // Log a message (plus a newline character) described by a format string with at // least one placeholder without flushing and with a reused timestamp. `format` -// is compile-time checked if CMAKE_CXX_STANDARD >= 14. +// is checked at compile time. #define BULK_LOG(format_, ...) \ do { \ if (Logging::enabled()) { \ diff -Nru ccache-4.2.1/src/macroskip.hpp ccache-4.5.1/src/macroskip.hpp --- ccache-4.2.1/src/macroskip.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/macroskip.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors +// Copyright (C) 2010-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,7 +18,7 @@ #pragma once -#include "system.hpp" +#include // A Boyer-Moore-Horspool skip table used for searching for the strings // "__TIME__", "__DATE__" and "__TIMESTAMP__". diff -Nru ccache-4.2.1/src/Manifest.cpp ccache-4.5.1/src/Manifest.cpp --- ccache-4.2.1/src/Manifest.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Manifest.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -19,37 +19,32 @@ #include "Manifest.hpp" #include "AtomicFile.hpp" -#include "CacheEntryReader.hpp" -#include "CacheEntryWriter.hpp" -#include "Checksum.hpp" #include "Config.hpp" #include "Context.hpp" #include "Digest.hpp" #include "File.hpp" #include "Hash.hpp" #include "Logging.hpp" -#include "Sloppiness.hpp" -#include "StdMakeUnique.hpp" #include "fmtmacros.hpp" #include "hashutil.hpp" +#include +#include +#include +#include +#include +#include +#include + +#include + // Manifest data format // ==================== // // Integers are big-endian. // -// ::=
::= -// -// ::= 4 bytes ("cCrS") -// ::= uint8_t -// ::= | -// ::= 0 (uint8_t) -// ::= 1 (uint8_t) -// ::= int8_t -// ::= uint64_t ; size of file if stored uncompressed -// ::= ; body is potentially -// ; compressed +// ::= +// ::= uint8_t // ::= * // ::= uint32_t // ::= @@ -65,52 +60,15 @@ // ::= int64_t ; status change time // ::= * // ::= uint32_t -// ::= * +// ::= * // ::= uint32_t // ::= uint32_t -// ::= Digest::size() bytes -// ::= -// ::= uint64_t ; XXH3 of content bytes -// -// Sketch of concrete layout: - -// 4 bytes -// 1 byte -// 1 byte -// 1 byte -// 8 bytes -// --- [potentially compressed from here] ------------------------------------- -// 4 bytes -// 2 bytes -// path_len bytes -// ... -// ---------------------------------------------------------------------------- -// 4 bytes -// 4 bytes -// Digest::size() bytes -// 8 bytes -// 8 bytes -// 8 bytes -// ... -// ---------------------------------------------------------------------------- -// 4 bytes -// 4 bytes -// 4 bytes -// ... -// Digest::size() bytes -// ... -// checksum 8 bytes -// -// -// Version history -// =============== -// -// 1: Introduced in ccache 3.0. (Files are always compressed with gzip.) -// 2: Introduced in ccache 4.0. +// ::= Digest::size() bytes using nonstd::nullopt; using nonstd::optional; +const uint8_t k_manifest_format_version = 0; const uint32_t k_max_manifest_entries = 100; const uint32_t k_max_manifest_file_info_entries = 10000; @@ -148,9 +106,9 @@ operator()(const FileInfo& file_info) const { static_assert(sizeof(FileInfo) == 48, "unexpected size"); // No padding. - Checksum checksum; - checksum.update(&file_info, sizeof(file_info)); - return checksum.digest(); + util::XXH3_64 hash; + hash.update(&file_info, sizeof(file_info)); + return hash.digest(); } }; @@ -163,14 +121,14 @@ // Indexes to file_infos. std::vector file_info_indexes; - // Name of the result. - Digest name; + // Key of the result. + Digest key; }; bool operator==(const ResultEntry& lhs, const ResultEntry& rhs) { - return lhs.file_info_indexes == rhs.file_info_indexes && lhs.name == rhs.name; + return lhs.file_info_indexes == rhs.file_info_indexes && lhs.key == rhs.key; } struct ManifestData @@ -181,12 +139,12 @@ // Information about referenced include files. std::vector file_infos; - // Result names plus references to include file infos. + // Result keys plus references to include file infos. std::vector results; bool add_result_entry( - const Digest& result_digest, + const Digest& result_key, const std::unordered_map& included_files, time_t time_of_compilation, bool save_timestamp) @@ -213,7 +171,7 @@ save_timestamp)); } - ResultEntry entry{std::move(file_info_indexes), result_digest}; + ResultEntry entry{std::move(file_info_indexes), result_key}; if (std::find(results.begin(), results.end(), entry) == results.end()) { results.push_back(std::move(entry)); return true; @@ -293,56 +251,63 @@ std::unique_ptr read_manifest(const std::string& path, FILE* dump_stream = nullptr) { - File file(path, "rb"); - if (!file) { - return {}; + FILE* file_stream; + File file; + if (path == "-") { + file_stream = stdin; + } else { + file = File(path, "rb"); + if (!file) { + return {}; + } + file_stream = file.get(); + } + + core::FileReader file_reader(file_stream); + core::CacheEntryReader reader(file_reader); + + if (dump_stream) { + reader.header().dump(dump_stream); } - CacheEntryReader reader(file.get(), Manifest::k_magic, Manifest::k_version); + const auto format_ver = reader.read_int(); + if (format_ver != k_manifest_format_version) { + throw core::Error("Unknown manifest format version: {}", format_ver); + } if (dump_stream) { - reader.dump_header(dump_stream); + PRINT(dump_stream, "Manifest format version: {}\n", format_ver); } auto mf = std::make_unique(); - uint32_t entry_count; - reader.read(entry_count); - for (uint32_t i = 0; i < entry_count; ++i) { - mf->files.emplace_back(); - auto& entry = mf->files.back(); - - uint16_t length; - reader.read(length); - entry.assign(length, 0); - reader.read(&entry[0], length); + const auto file_count = reader.read_int(); + for (uint32_t i = 0; i < file_count; ++i) { + mf->files.push_back(reader.read_str(reader.read_int())); } - reader.read(entry_count); - for (uint32_t i = 0; i < entry_count; ++i) { + const auto file_info_count = reader.read_int(); + for (uint32_t i = 0; i < file_info_count; ++i) { mf->file_infos.emplace_back(); auto& entry = mf->file_infos.back(); - reader.read(entry.index); + reader.read_int(entry.index); reader.read(entry.digest.bytes(), Digest::size()); - reader.read(entry.fsize); - reader.read(entry.mtime); - reader.read(entry.ctime); + reader.read_int(entry.fsize); + reader.read_int(entry.mtime); + reader.read_int(entry.ctime); } - reader.read(entry_count); - for (uint32_t i = 0; i < entry_count; ++i) { + const auto result_count = reader.read_int(); + for (uint32_t i = 0; i < result_count; ++i) { mf->results.emplace_back(); auto& entry = mf->results.back(); - uint32_t file_info_count; - reader.read(file_info_count); - for (uint32_t j = 0; j < file_info_count; ++j) { - uint32_t file_info_index; - reader.read(file_info_index); - entry.file_info_indexes.push_back(file_info_index); + const auto file_info_index_count = reader.read_int(); + for (uint32_t j = 0; j < file_info_index_count; ++j) { + entry.file_info_indexes.push_back(reader.read_int()); } - reader.read(entry.name.bytes(), Digest::size()); + reader.read(entry.key.bytes(), Digest::size()); } reader.finalize(); @@ -355,6 +320,7 @@ const ManifestData& mf) { uint64_t payload_size = 0; + payload_size += 1; // format_ver payload_size += 4; // n_files for (const auto& file : mf.files) { payload_size += 2 + file.length(); @@ -369,34 +335,39 @@ } AtomicFile atomic_manifest_file(path, AtomicFile::Mode::binary); - CacheEntryWriter writer(atomic_manifest_file.stream(), - Manifest::k_magic, - Manifest::k_version, - Compression::type_from_config(config), - Compression::level_from_config(config), - payload_size); - writer.write(mf.files.size()); + core::FileWriter file_writer(atomic_manifest_file.stream()); + core::CacheEntryHeader header(core::CacheEntryType::manifest, + compression::type_from_config(config), + compression::level_from_config(config), + time(nullptr), + CCACHE_VERSION, + config.namespace_()); + header.set_entry_size_from_payload_size(payload_size); + + core::CacheEntryWriter writer(file_writer, header); + writer.write_int(k_manifest_format_version); + writer.write_int(mf.files.size()); for (const auto& file : mf.files) { - writer.write(file.length()); - writer.write(file.data(), file.length()); + writer.write_int(file.length()); + writer.write_str(file); } - writer.write(mf.file_infos.size()); + writer.write_int(mf.file_infos.size()); for (const auto& file_info : mf.file_infos) { - writer.write(file_info.index); + writer.write_int(file_info.index); writer.write(file_info.digest.bytes(), Digest::size()); - writer.write(file_info.fsize); - writer.write(file_info.mtime); - writer.write(file_info.ctime); + writer.write_int(file_info.fsize); + writer.write_int(file_info.mtime); + writer.write_int(file_info.ctime); } - writer.write(mf.results.size()); + writer.write_int(mf.results.size()); for (const auto& result : mf.results) { - writer.write(result.file_info_indexes.size()); + writer.write_int(result.file_info_indexes.size()); for (auto index : result.file_info_indexes) { - writer.write(index); + writer.write_int(index); } - writer.write(result.name.bytes(), Digest::size()); + writer.write(result.key.bytes(), Digest::size()); } writer.finalize(); @@ -443,8 +414,9 @@ return false; } - if (ctx.config.sloppiness() & SLOPPY_FILE_STAT_MATCHES) { - if (!(ctx.config.sloppiness() & SLOPPY_FILE_STAT_MATCHES_CTIME)) { + if (ctx.config.sloppiness().is_enabled(core::Sloppy::file_stat_matches)) { + if (!(ctx.config.sloppiness().is_enabled( + core::Sloppy::file_stat_matches_ctime))) { if (fi.mtime == fs.mtime && fi.ctime == fs.ctime) { LOG("mtime/ctime hit for {}", path); continue; @@ -493,21 +465,18 @@ const uint8_t k_magic[4] = {'c', 'C', 'm', 'F'}; const uint8_t k_version = 2; -// Try to get the result name from a manifest file. Returns nullopt on failure. +// Try to get the result key from a manifest file. Returns nullopt on failure. optional get(const Context& ctx, const std::string& path) { std::unique_ptr mf; try { mf = read_manifest(path); - if (mf) { - // Update modification timestamp to save files from LRU cleanup. - Util::update_mtime(path); - } else { + if (!mf) { LOG_RAW("No such manifest file"); return nullopt; } - } catch (const Error& e) { + } catch (const core::Error& e) { LOG("Error: {}", e.what()); return nullopt; } @@ -519,19 +488,19 @@ for (uint32_t i = mf->results.size(); i > 0; i--) { if (verify_result( ctx, *mf, mf->results[i - 1], stated_files, hashed_files)) { - return mf->results[i - 1].name; + return mf->results[i - 1].key; } } return nullopt; } -// Put the result name into a manifest file given a set of included files. +// Put the result key into a manifest file given a set of included files. // Returns true on success, otherwise false. bool put(const Config& config, const std::string& path, - const Digest& result_name, + const Digest& result_key, const std::unordered_map& included_files, time_t time_of_compilation, @@ -548,7 +517,7 @@ // Manifest file didn't exist. mf = std::make_unique(); } - } catch (const Error& e) { + } catch (const core::Error& e) { LOG("Error: {}", e.what()); // Manifest file was corrupt, ignore. mf = std::make_unique(); @@ -578,13 +547,13 @@ } bool added = mf->add_result_entry( - result_name, included_files, time_of_compilation, save_timestamp); + result_key, included_files, time_of_compilation, save_timestamp); if (added) { try { write_manifest(config, path, *mf); return true; - } catch (const Error& e) { + } catch (const core::Error& e) { LOG("Error: {}", e.what()); } } else { @@ -599,7 +568,7 @@ std::unique_ptr mf; try { mf = read_manifest(path, stream); - } catch (const Error& e) { + } catch (const core::Error& e) { PRINT(stream, "Error: {}\n", e.what()); return false; } @@ -630,7 +599,7 @@ PRINT(stream, " {}", file_info_index); } PRINT_RAW(stream, "\n"); - PRINT(stream, " Name: {}\n", mf->results[i].name.to_string()); + PRINT(stream, " Key: {}\n", mf->results[i].key.to_string()); } return true; diff -Nru ccache-4.2.1/src/Manifest.hpp ccache-4.5.1/src/Manifest.hpp --- ccache-4.2.1/src/Manifest.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Manifest.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors +// Copyright (C) 2009-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,10 +18,11 @@ #pragma once -#include "system.hpp" - #include "third_party/nonstd/optional.hpp" +#include +#include +#include #include #include @@ -38,7 +39,7 @@ nonstd::optional get(const Context& ctx, const std::string& path); bool put(const Config& config, const std::string& path, - const Digest& result_name, + const Digest& result_key, const std::unordered_map& included_files, time_t time_of_compilation, bool save_timestamp); diff -Nru ccache-4.2.1/src/MiniTrace.cpp ccache-4.5.1/src/MiniTrace.cpp --- ccache-4.2.1/src/MiniTrace.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/MiniTrace.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -16,62 +16,67 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -#include "system.hpp" +#include "MiniTrace.hpp" -#ifdef MTR_ENABLED +#include "ArgsInfo.hpp" +#include "TemporaryFile.hpp" +#include "Util.hpp" +#include "fmtmacros.hpp" -# include "ArgsInfo.hpp" -# include "MiniTrace.hpp" -# include "TemporaryFile.hpp" -# include "Util.hpp" -# include "fmtmacros.hpp" - -# ifdef HAVE_SYS_TIME_H -# include -# endif +#include + +#ifdef HAVE_SYS_TIME_H +# include +#endif + +#ifdef HAVE_UNISTD_H +# include +#endif namespace { std::string get_system_tmp_dir() { -# ifndef _WIN32 +#ifndef _WIN32 const char* tmpdir = getenv("TMPDIR"); if (tmpdir) { return tmpdir; } -# else +#else static char dirbuf[PATH_MAX]; DWORD retval = GetTempPath(PATH_MAX, dirbuf); if (retval > 0 && retval < PATH_MAX) { return dirbuf; } -# endif +#endif return "/tmp"; } double time_seconds() { -# ifdef HAVE_GETTIMEOFDAY +#ifdef HAVE_GETTIMEOFDAY struct timeval tv; gettimeofday(&tv, nullptr); return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -# else +#else return (double)time(nullptr); -# endif +#endif } } // namespace MiniTrace::MiniTrace(const ArgsInfo& args_info) - : m_args_info(args_info), m_trace_id(reinterpret_cast(getpid())) + : m_args_info(args_info), + m_trace_id(reinterpret_cast(getpid())) { TemporaryFile tmp_file(get_system_tmp_dir() + "/ccache-trace"); m_tmp_trace_file = tmp_file.path; mtr_init(m_tmp_trace_file.c_str()); - MTR_INSTANT_C("", "", "time", FMT("{:f}", time_seconds()).c_str()); + m_start_time = FMT("{:f}", time_seconds()); + MTR_INSTANT_C("", "", "time", m_start_time.c_str()); MTR_META_PROCESS_NAME("ccache"); MTR_START("program", "ccache", m_trace_id); } @@ -87,5 +92,3 @@ } Util::unlink_tmp(m_tmp_trace_file); } - -#endif diff -Nru ccache-4.2.1/src/MiniTrace.hpp ccache-4.5.1/src/MiniTrace.hpp --- ccache-4.2.1/src/MiniTrace.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/MiniTrace.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,13 +18,9 @@ #pragma once -#include "system.hpp" - #include "third_party/minitrace.h" -#ifdef MTR_ENABLED - -# include +#include struct ArgsInfo; @@ -38,6 +34,5 @@ const ArgsInfo& m_args_info; const void* const m_trace_id; std::string m_tmp_trace_file; + std::string m_start_time; }; - -#endif diff -Nru ccache-4.2.1/src/NullCompressor.cpp ccache-4.5.1/src/NullCompressor.cpp --- ccache-4.2.1/src/NullCompressor.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/NullCompressor.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,47 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "NullCompressor.hpp" - -#include "exceptions.hpp" - -NullCompressor::NullCompressor(FILE* stream) : m_stream(stream) -{ -} - -int8_t -NullCompressor::actual_compression_level() const -{ - return 0; -} - -void -NullCompressor::write(const void* data, size_t count) -{ - if (fwrite(data, 1, count, m_stream) != count) { - throw Error("failed to write to uncompressed stream"); - } -} - -void -NullCompressor::finalize() -{ - if (fflush(m_stream) != 0) { - throw Error("failed to finalize uncompressed stream"); - } -} diff -Nru ccache-4.2.1/src/NullCompressor.hpp ccache-4.5.1/src/NullCompressor.hpp --- ccache-4.2.1/src/NullCompressor.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/NullCompressor.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,40 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Compressor.hpp" -#include "NonCopyable.hpp" - -// A compressor of an uncompressed stream. -class NullCompressor : public Compressor, NonCopyable -{ -public: - // Parameters: - // - stream: The file to write data to. - explicit NullCompressor(FILE* stream); - - int8_t actual_compression_level() const override; - void write(const void* data, size_t count) override; - void finalize() override; - -private: - FILE* m_stream; -}; diff -Nru ccache-4.2.1/src/NullDecompressor.cpp ccache-4.5.1/src/NullDecompressor.cpp --- ccache-4.2.1/src/NullDecompressor.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/NullDecompressor.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,41 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "NullDecompressor.hpp" - -#include "exceptions.hpp" - -NullDecompressor::NullDecompressor(FILE* stream) : m_stream(stream) -{ -} - -void -NullDecompressor::read(void* data, size_t count) -{ - if (fread(data, count, 1, m_stream) != 1) { - throw Error("failed to read from uncompressed stream"); - } -} - -void -NullDecompressor::finalize() -{ - if (fgetc(m_stream) != EOF) { - throw Error("garbage data at end of uncompressed stream"); - } -} diff -Nru ccache-4.2.1/src/NullDecompressor.hpp ccache-4.5.1/src/NullDecompressor.hpp --- ccache-4.2.1/src/NullDecompressor.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/NullDecompressor.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,39 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Decompressor.hpp" -#include "NonCopyable.hpp" - -// A decompressor of an uncompressed stream. -class NullDecompressor : public Decompressor, NonCopyable -{ -public: - // Parameters: - // - stream: The file to read data from. - explicit NullDecompressor(FILE* stream); - - void read(void* data, size_t count) override; - void finalize() override; - -private: - FILE* m_stream; -}; diff -Nru ccache-4.2.1/src/ProgressBar.cpp ccache-4.5.1/src/ProgressBar.cpp --- ccache-4.2.1/src/ProgressBar.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ProgressBar.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,9 +20,12 @@ #include "fmtmacros.hpp" +#include + #include "third_party/fmt/core.h" -#ifndef _WIN32 +#ifdef _WIN32 +#else # include #endif @@ -30,6 +33,10 @@ # include #endif +#ifdef HAVE_UNISTD_H +# include +#endif + #include namespace { diff -Nru ccache-4.2.1/src/ProgressBar.hpp ccache-4.5.1/src/ProgressBar.hpp --- ccache-4.2.1/src/ProgressBar.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ProgressBar.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,8 @@ #pragma once -#include "system.hpp" - +#include +#include #include class ProgressBar diff -Nru ccache-4.2.1/src/Result.cpp ccache-4.5.1/src/Result.cpp --- ccache-4.2.1/src/Result.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Result.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,19 +19,33 @@ #include "Result.hpp" #include "AtomicFile.hpp" -#include "CacheEntryReader.hpp" -#include "CacheEntryWriter.hpp" #include "Config.hpp" #include "Context.hpp" #include "Fd.hpp" #include "File.hpp" #include "Logging.hpp" #include "Stat.hpp" -#include "Statistic.hpp" #include "Util.hpp" -#include "exceptions.hpp" #include "fmtmacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #include // Result data format @@ -39,17 +53,8 @@ // // Integers are big-endian. // -// ::=
-//
::= -// -// ::= 4 bytes ("cCrS") -// ::= uint8_t -// ::= | -// ::= 0 (uint8_t) -// ::= 1 (uint8_t) -// ::= int8_t -// ::= uint64_t ; size of file if stored uncompressed -// ::= * ; potentially compressed +// ::= * +// ::= uint8_t // ::= uint8_t // ::= | // ::= @@ -63,32 +68,6 @@ // ::= uint64_t // ::= // ::= uint64_t ; XXH3 of content bytes -// -// Sketch of concrete layout: -// -// 4 bytes -// 1 byte -// 1 byte -// 1 byte -// 8 bytes -// --- [potentially compressed from here] ------------------------------------- -// 1 byte -// 1 byte -// 1 byte -// 8 bytes -// data_len bytes -// ... -// 1 byte -// 1 byte -// key_len bytes -// ... -// checksum 8 bytes -// -// -// Version history -// =============== -// -// 1: Introduced in ccache 4.0. using nonstd::nullopt; using nonstd::optional; @@ -96,6 +75,8 @@ namespace { +const uint8_t k_result_format_version = 0; + // File data stored inside the result file. const uint8_t k_embedded_file_marker = 0; @@ -182,7 +163,7 @@ { const auto& output_obj = ctx.args_info.output_obj; const std::string abs_output_obj = - Util::is_absolute_path(output_obj) + util::is_absolute_path(output_obj) ? output_obj : FMT("{}/{}", ctx.apparent_cwd, output_obj); std::string hashified_obj = abs_output_obj; @@ -196,6 +177,14 @@ return Util::change_extension(ctx.args_info.output_obj, ".gcno"); } +FileSizeAndCountDiff& +FileSizeAndCountDiff::operator+=(const FileSizeAndCountDiff& other) +{ + size_kibibyte += other.size_kibibyte; + count += other.count; + return *this; +} + Result::Reader::Reader(const std::string& result_path) : m_result_path(result_path) { @@ -212,7 +201,7 @@ } else { return "No such result file"; } - } catch (const Error& e) { + } catch (const core::Error& e) { return e.what(); } } @@ -220,18 +209,31 @@ bool Reader::read_result(Consumer& consumer) { - File file(m_result_path, "rb"); - if (!file) { - // Cache miss. - return false; + FILE* file_stream; + File file; + if (m_result_path == "-") { + file_stream = stdin; + } else { + file = File(m_result_path, "rb"); + if (!file) { + // Cache miss. + return false; + } + file_stream = file.get(); } - CacheEntryReader cache_entry_reader(file.get(), k_magic, k_version); + core::FileReader file_reader(file_stream); + core::CacheEntryReader cache_entry_reader(file_reader); + + const auto result_format_version = cache_entry_reader.read_int(); + if (result_format_version != k_result_format_version) { + throw core::Error("Unknown result format version: {}", + result_format_version); + } - consumer.on_header(cache_entry_reader); + consumer.on_header(cache_entry_reader, result_format_version); - uint8_t n_entries; - cache_entry_reader.read(n_entries); + const auto n_entries = cache_entry_reader.read_int(); uint32_t i; for (i = 0; i < n_entries; ++i) { @@ -239,7 +241,7 @@ } if (i != n_entries) { - throw Error("Too few entries (read {}, expected {})", i, n_entries); + throw core::Error("Too few entries (read {}, expected {})", i, n_entries); } cache_entry_reader.finalize(); @@ -247,12 +249,11 @@ } void -Reader::read_entry(CacheEntryReader& cache_entry_reader, +Reader::read_entry(core::CacheEntryReader& cache_entry_reader, uint32_t entry_number, Reader::Consumer& consumer) { - uint8_t marker; - cache_entry_reader.read(marker); + const auto marker = cache_entry_reader.read_int(); switch (marker) { case k_embedded_file_marker: @@ -260,20 +261,17 @@ break; default: - throw Error("Unknown entry type: {}", marker); + throw core::Error("Unknown entry type: {}", marker); } - UnderlyingFileTypeInt type; - cache_entry_reader.read(type); - FileType file_type = FileType(type); - - uint64_t file_len; - cache_entry_reader.read(file_len); + const auto type = cache_entry_reader.read_int(); + const auto file_type = FileType(type); + const auto file_len = cache_entry_reader.read_int(); if (marker == k_embedded_file_marker) { consumer.on_entry_start(entry_number, file_type, file_len, nullopt); - uint8_t buf[READ_BUFFER_SIZE]; + uint8_t buf[CCACHE_READ_BUFFER_SIZE]; size_t remain = file_len; while (remain > 0) { size_t n = std::min(remain, sizeof(buf)); @@ -287,10 +285,11 @@ auto raw_path = get_raw_file_path(m_result_path, entry_number); auto st = Stat::stat(raw_path, Stat::OnError::throw_error); if (st.size() != file_len) { - throw Error("Bad file size of {} (actual {} bytes, expected {} bytes)", - raw_path, - st.size(), - file_len); + throw core::Error( + "Bad file size of {} (actual {} bytes, expected {} bytes)", + raw_path, + st.size(), + file_len); } consumer.on_entry_start(entry_number, file_type, file_len, raw_path); @@ -300,7 +299,8 @@ } Writer::Writer(Context& ctx, const std::string& result_path) - : m_ctx(ctx), m_result_path(result_path) + : m_ctx(ctx), + m_result_path(result_path) { } @@ -310,21 +310,22 @@ m_entries_to_write.emplace_back(file_type, file_path); } -optional +nonstd::expected Writer::finalize() { try { - do_finalize(); - return nullopt; - } catch (const Error& e) { - return e.what(); + return do_finalize(); + } catch (const core::Error& e) { + return nonstd::make_unexpected(e.what()); } } -void +FileSizeAndCountDiff Writer::do_finalize() { + FileSizeAndCountDiff file_size_and_count_diff{0, 0}; uint64_t payload_size = 0; + payload_size += 1; // format_ver payload_size += 1; // n_entries for (const auto& pair : m_entries_to_write) { const auto& path = pair.second; @@ -337,20 +338,25 @@ } AtomicFile atomic_result_file(m_result_path, AtomicFile::Mode::binary); - CacheEntryWriter writer(atomic_result_file.stream(), - k_magic, - k_version, - Compression::type_from_config(m_ctx.config), - Compression::level_from_config(m_ctx.config), - payload_size); + core::CacheEntryHeader header(core::CacheEntryType::result, + compression::type_from_config(m_ctx.config), + compression::level_from_config(m_ctx.config), + time(nullptr), + CCACHE_VERSION, + m_ctx.config.namespace_()); + header.set_entry_size_from_payload_size(payload_size); + + core::FileWriter file_writer(atomic_result_file.stream()); + core::CacheEntryWriter writer(file_writer, header); - writer.write(m_entries_to_write.size()); + writer.write_int(k_result_format_version); + writer.write_int(m_entries_to_write.size()); uint32_t entry_number = 0; for (const auto& pair : m_entries_to_write) { const auto file_type = pair.first; const auto& path = pair.second; - LOG("Storing result {}", path); + LOG("Storing result file {}", path); const bool store_raw = should_store_raw_file(m_ctx.config, file_type); uint64_t file_size = Stat::stat(path, Stat::OnError::throw_error).size(); @@ -362,13 +368,13 @@ file_size, path); - writer.write(store_raw ? k_raw_file_marker - : k_embedded_file_marker); - writer.write(UnderlyingFileTypeInt(file_type)); - writer.write(file_size); + writer.write_int(store_raw ? k_raw_file_marker + : k_embedded_file_marker); + writer.write_int(UnderlyingFileTypeInt(file_type)); + writer.write_int(file_size); if (store_raw) { - write_raw_file_entry(path, entry_number); + file_size_and_count_diff += write_raw_file_entry(path, entry_number); } else { write_embedded_file_entry(writer, path, file_size); } @@ -378,38 +384,40 @@ writer.finalize(); atomic_result_file.commit(); + + return file_size_and_count_diff; } void -Result::Writer::write_embedded_file_entry(CacheEntryWriter& writer, +Result::Writer::write_embedded_file_entry(core::CacheEntryWriter& writer, const std::string& path, uint64_t file_size) { Fd file(open(path.c_str(), O_RDONLY | O_BINARY)); if (!file) { - throw Error("Failed to open {} for reading", path); + throw core::Error("Failed to open {} for reading", path); } uint64_t remain = file_size; while (remain > 0) { - uint8_t buf[READ_BUFFER_SIZE]; + uint8_t buf[CCACHE_READ_BUFFER_SIZE]; size_t n = std::min(remain, static_cast(sizeof(buf))); - ssize_t bytes_read = read(*file, buf, n); + auto bytes_read = read(*file, buf, n); if (bytes_read == -1) { if (errno == EINTR) { continue; } - throw Error("Error reading from {}: {}", path, strerror(errno)); + throw core::Error("Error reading from {}: {}", path, strerror(errno)); } if (bytes_read == 0) { - throw Error("Error reading from {}: end of file", path); + throw core::Error("Error reading from {}: end of file", path); } writer.write(buf, bytes_read); remain -= bytes_read; } } -void +FileSizeAndCountDiff Result::Writer::write_raw_file_entry(const std::string& path, uint32_t entry_number) { @@ -417,16 +425,15 @@ const auto old_stat = Stat::stat(raw_file); try { Util::clone_hard_link_or_copy_file(m_ctx, path, raw_file, true); - } catch (Error& e) { - throw Error( + } catch (core::Error& e) { + throw core::Error( "Failed to store {} as raw file {}: {}", path, raw_file, e.what()); } const auto new_stat = Stat::stat(raw_file); - m_ctx.counter_updates.increment( - Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(old_stat, new_stat)); - m_ctx.counter_updates.increment(Statistic::files_in_cache, - (new_stat ? 1 : 0) - (old_stat ? 1 : 0)); + return { + Util::size_change_kibibyte(old_stat, new_stat), + (new_stat ? 1 : 0) - (old_stat ? 1 : 0), + }; } } // namespace Result diff -Nru ccache-4.2.1/src/ResultDumper.cpp ccache-4.5.1/src/ResultDumper.cpp --- ccache-4.2.1/src/ResultDumper.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ResultDumper.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,11 +18,12 @@ #include "ResultDumper.hpp" -#include "CacheEntryReader.hpp" #include "Context.hpp" #include "Logging.hpp" #include "fmtmacros.hpp" +#include + using nonstd::optional; ResultDumper::ResultDumper(FILE* stream) : m_stream(stream) @@ -30,9 +31,11 @@ } void -ResultDumper::on_header(CacheEntryReader& cache_entry_reader) +ResultDumper::on_header(core::CacheEntryReader& cache_entry_reader, + const uint8_t result_format_version) { - cache_entry_reader.dump_header(m_stream); + cache_entry_reader.header().dump(m_stream); + PRINT(m_stream, "Result format version: {}\n", result_format_version); } void diff -Nru ccache-4.2.1/src/ResultDumper.hpp ccache-4.5.1/src/ResultDumper.hpp --- ccache-4.2.1/src/ResultDumper.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ResultDumper.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,17 +18,19 @@ #pragma once -#include "system.hpp" - #include "Result.hpp" +#include +#include + // This class dumps information about the result entry to `stream`. class ResultDumper : public Result::Reader::Consumer { public: ResultDumper(FILE* stream); - void on_header(CacheEntryReader& cache_entry_reader) override; + void on_header(core::CacheEntryReader& cache_entry_reader, + uint8_t result_format_version) override; void on_entry_start(uint32_t entry_number, Result::FileType file_type, uint64_t file_len, diff -Nru ccache-4.2.1/src/ResultExtractor.cpp ccache-4.5.1/src/ResultExtractor.cpp --- ccache-4.2.1/src/ResultExtractor.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ResultExtractor.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -21,13 +21,21 @@ #include "Util.hpp" #include "fmtmacros.hpp" +#include +#include + +#include +#include +#include + ResultExtractor::ResultExtractor(const std::string& directory) : m_directory(directory) { } void -ResultExtractor::on_header(CacheEntryReader& /*cache_entry_reader*/) +ResultExtractor::on_header(core::CacheEntryReader& /*cache_entry_reader*/, + const uint8_t /*result_format_version*/) { } @@ -51,14 +59,14 @@ m_dest_fd = Fd( open(m_dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); if (!m_dest_fd) { - throw Error( + throw core::Error( "Failed to open {} for writing: {}", m_dest_path, strerror(errno)); } } else { try { Util::copy_file(*raw_file, m_dest_path, false); - } catch (Error& e) { - throw Error( + } catch (core::Error& e) { + throw core::Error( "Failed to copy {} to {}: {}", *raw_file, m_dest_path, e.what()); } } @@ -71,8 +79,8 @@ try { Util::write_fd(*m_dest_fd, data, size); - } catch (Error& e) { - throw Error("Failed to write to {}: {}", m_dest_path, e.what()); + } catch (core::Error& e) { + throw core::Error("Failed to write to {}: {}", m_dest_path, e.what()); } } diff -Nru ccache-4.2.1/src/ResultExtractor.hpp ccache-4.5.1/src/ResultExtractor.hpp --- ccache-4.2.1/src/ResultExtractor.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ResultExtractor.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "Fd.hpp" #include "Result.hpp" @@ -31,7 +29,8 @@ public: ResultExtractor(const std::string& directory); - void on_header(CacheEntryReader& cache_entry_reader) override; + void on_header(core::CacheEntryReader& cache_entry_reader, + uint8_t result_format_version) override; void on_entry_start(uint32_t entry_number, Result::FileType file_type, uint64_t file_len, diff -Nru ccache-4.2.1/src/Result.hpp ccache-4.5.1/src/Result.hpp --- ccache-4.2.1/src/Result.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Result.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,16 +18,21 @@ #pragma once -#include "system.hpp" - +#include "third_party/nonstd/expected.hpp" #include "third_party/nonstd/optional.hpp" +#include #include #include #include +namespace core { + class CacheEntryReader; class CacheEntryWriter; + +} // namespace core + class Context; namespace Result { @@ -64,7 +69,7 @@ // Diagnostics output file specified by --serialize-diagnostics. diagnostic = 5, - // DWARF object file geenrated by -gsplit-dwarf, i.e. output file but with a + // DWARF object file generated by -gsplit-dwarf, i.e. output file but with a // .dwo extension. dwarf_object = 6, @@ -79,6 +84,14 @@ std::string gcno_file_in_mangled_form(const Context& ctx); std::string gcno_file_in_unmangled_form(const Context& ctx); +struct FileSizeAndCountDiff +{ + int64_t size_kibibyte; + int64_t count; + + FileSizeAndCountDiff& operator+=(const FileSizeAndCountDiff& other); +}; + // This class knows how to read a result cache entry. class Reader { @@ -90,7 +103,8 @@ public: virtual ~Consumer() = default; - virtual void on_header(CacheEntryReader& cache_entry_reader) = 0; + virtual void on_header(core::CacheEntryReader& cache_entry_reader, + uint8_t result_format_version) = 0; virtual void on_entry_start(uint32_t entry_number, FileType file_type, uint64_t file_len, @@ -106,7 +120,7 @@ const std::string m_result_path; bool read_result(Consumer& consumer); - void read_entry(CacheEntryReader& cache_entry_reader, + void read_entry(core::CacheEntryReader& cache_entry_reader, uint32_t entry_number, Reader::Consumer& consumer); }; @@ -121,18 +135,19 @@ void write(FileType file_type, const std::string& file_path); // Write registered files to the result. Returns an error message on error. - nonstd::optional finalize(); + nonstd::expected finalize(); private: Context& m_ctx; const std::string m_result_path; std::vector> m_entries_to_write; - void do_finalize(); - static void write_embedded_file_entry(CacheEntryWriter& writer, + FileSizeAndCountDiff do_finalize(); + static void write_embedded_file_entry(core::CacheEntryWriter& writer, const std::string& path, uint64_t file_size); - void write_raw_file_entry(const std::string& path, uint32_t entry_number); + FileSizeAndCountDiff write_raw_file_entry(const std::string& path, + uint32_t entry_number); }; } // namespace Result diff -Nru ccache-4.2.1/src/ResultRetriever.cpp ccache-4.5.1/src/ResultRetriever.cpp --- ccache-4.2.1/src/ResultRetriever.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ResultRetriever.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -22,15 +22,28 @@ #include "Depfile.hpp" #include "Logging.hpp" +#include +#include + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + using Result::FileType; ResultRetriever::ResultRetriever(Context& ctx, bool rewrite_dependency_target) - : m_ctx(ctx), m_rewrite_dependency_target(rewrite_dependency_target) + : m_ctx(ctx), + m_rewrite_dependency_target(rewrite_dependency_target) { } void -ResultRetriever::on_header(CacheEntryReader& /*cache_entry_reader*/) +ResultRetriever::on_header(core::CacheEntryReader& /*cache_entry_reader*/, + const uint8_t /*result_format_version*/) { } @@ -118,7 +131,7 @@ m_dest_fd = Fd( open(dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); if (!m_dest_fd) { - throw Error( + throw core::Error( "Failed to open {} for writing: {}", dest_path, strerror(errno)); } m_dest_path = dest_path; @@ -136,8 +149,8 @@ } else if (m_dest_fd) { try { Util::write_fd(*m_dest_fd, data, size); - } catch (Error& e) { - throw Error("Failed to write to {}: {}", m_dest_path, e.what()); + } catch (core::Error& e) { + throw core::Error("Failed to write to {}: {}", m_dest_path, e.what()); } } } @@ -178,7 +191,7 @@ Util::write_fd(*m_dest_fd, m_dest_data.data() + start_pos, m_dest_data.length() - start_pos); - } catch (Error& e) { - throw Error("Failed to write to {}: {}", m_dest_path, e.what()); + } catch (core::Error& e) { + throw core::Error("Failed to write to {}: {}", m_dest_path, e.what()); } } diff -Nru ccache-4.2.1/src/ResultRetriever.hpp ccache-4.5.1/src/ResultRetriever.hpp --- ccache-4.2.1/src/ResultRetriever.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/ResultRetriever.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "Fd.hpp" #include "Result.hpp" @@ -31,7 +29,8 @@ public: ResultRetriever(Context& ctx, bool rewrite_dependency_target); - void on_header(CacheEntryReader& cache_entry_reader) override; + void on_header(core::CacheEntryReader& cache_entry_reader, + uint8_t result_format_version) override; void on_entry_start(uint32_t entry_number, Result::FileType file_type, uint64_t file_len, @@ -41,7 +40,7 @@ private: Context& m_ctx; - Result::FileType m_dest_file_type; + Result::FileType m_dest_file_type{}; Fd m_dest_fd; std::string m_dest_path; diff -Nru ccache-4.2.1/src/SignalHandler.cpp ccache-4.5.1/src/SignalHandler.cpp --- ccache-4.2.1/src/SignalHandler.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/SignalHandler.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -24,6 +24,9 @@ # include "assertions.hpp" # include // NOLINT: sigaddset et al are defined in signal.h +# include +# include +# include namespace { @@ -68,6 +71,8 @@ # ifdef SIGQUIT register_signal_handler(SIGQUIT); # endif + + signal(SIGPIPE, SIG_IGN); // NOLINT: This is no error, clang-tidy } SignalHandler::~SignalHandler() diff -Nru ccache-4.2.1/src/SignalHandler.hpp ccache-4.5.1/src/SignalHandler.hpp --- ccache-4.2.1/src/SignalHandler.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/SignalHandler.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - class Context; class SignalHandler diff -Nru ccache-4.2.1/src/Sloppiness.hpp ccache-4.5.1/src/Sloppiness.hpp --- ccache-4.2.1/src/Sloppiness.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Sloppiness.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,41 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -enum Sloppiness { - SLOPPY_INCLUDE_FILE_MTIME = 1 << 0, - SLOPPY_INCLUDE_FILE_CTIME = 1 << 1, - SLOPPY_TIME_MACROS = 1 << 2, - SLOPPY_PCH_DEFINES = 1 << 3, - // Allow us to match files based on their stats (size, mtime, ctime), without - // looking at their contents. - SLOPPY_FILE_STAT_MATCHES = 1 << 4, - // Allow us to not include any system headers in the manifest include files, - // similar to -MM versus -M for dependencies. - SLOPPY_SYSTEM_HEADERS = 1 << 5, - // Allow us to ignore ctimes when comparing file stats, so we can fake mtimes - // if we want to (it is much harder to fake ctimes, requires changing clock) - SLOPPY_FILE_STAT_MATCHES_CTIME = 1 << 6, - // Allow us to not include the -index-store-path option in the manifest hash. - SLOPPY_CLANG_INDEX_STORE = 1 << 7, - // Ignore locale settings. - SLOPPY_LOCALE = 1 << 8, - // Allow caching even if -fmodules is used. - SLOPPY_MODULES = 1 << 9, -}; diff -Nru ccache-4.2.1/src/Stat.cpp ccache-4.5.1/src/Stat.cpp --- ccache-4.2.1/src/Stat.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Stat.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,12 +18,16 @@ #include "Stat.hpp" -#ifdef _WIN32 -# include "third_party/win32/winerror_to_errno.h" -#endif - #include "Finalizer.hpp" #include "Logging.hpp" +#include "Win32Util.hpp" + +#include +#include + +#ifdef _WIN32 +# include +#endif namespace { @@ -124,6 +128,11 @@ } if (handle == INVALID_HANDLE_VALUE) { + if (GetLastError() == ERROR_ACCESS_DENIED + && Win32Util::get_last_ntstatus() == STATUS_DELETE_PENDING) { + // Treat a 'pending delete' as a nonexistent file. + SetLastError(ERROR_FILE_NOT_FOUND); + } return false; } @@ -203,7 +212,7 @@ } else { m_errno = errno; if (on_error == OnError::throw_error) { - throw Error("failed to stat {}: {}", path, strerror(errno)); + throw core::Error("failed to stat {}: {}", path, strerror(errno)); } if (on_error == OnError::log) { LOG("Failed to stat {}: {}", path, strerror(errno)); diff -Nru ccache-4.2.1/src/Stat.hpp ccache-4.5.1/src/Stat.hpp --- ccache-4.2.1/src/Stat.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Stat.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,12 +18,44 @@ #pragma once -#include "system.hpp" +#include -#include "exceptions.hpp" +#include +#include +#include #include +#ifdef _WIN32 +# ifndef S_IFIFO +# define S_IFIFO 0x1000 +# endif +# ifndef S_IFBLK +# define S_IFBLK 0x6000 +# endif +# ifndef S_IFLNK +# define S_IFLNK 0xA000 +# endif +# ifndef S_ISREG +# define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) +# endif +# ifndef S_ISDIR +# define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) +# endif +# ifndef S_ISFIFO +# define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO) +# endif +# ifndef S_ISCHR +# define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR) +# endif +# ifndef S_ISLNK +# define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK) +# endif +# ifndef S_ISBLK +# define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK) +# endif +#endif + class Stat { public: @@ -94,6 +126,7 @@ dev_t device() const; ino_t inode() const; mode_t mode() const; + time_t atime() const; time_t ctime() const; time_t mtime() const; uint64_t size() const; @@ -109,6 +142,7 @@ uint32_t reparse_tag() const; #endif + timespec atim() const; timespec ctim() const; timespec mtim() const; @@ -165,6 +199,12 @@ } inline time_t +Stat::atime() const +{ + return atim().tv_sec; +} + +inline time_t Stat::ctime() const { return ctim().tv_sec; @@ -225,6 +265,16 @@ #endif inline timespec +Stat::atim() const +{ +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_ATIM) + return m_stat.st_atim; +#else + return {m_stat.st_atime, 0}; +#endif +} + +inline timespec Stat::ctim() const { #if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_CTIM) diff -Nru ccache-4.2.1/src/Statistic.hpp ccache-4.5.1/src/Statistic.hpp --- ccache-4.2.1/src/Statistic.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Statistic.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,58 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -// Statistics fields in storage order. -enum class Statistic { - none = 0, - compiler_produced_stdout = 1, - compile_failed = 2, - internal_error = 3, - cache_miss = 4, - preprocessor_error = 5, - could_not_find_compiler = 6, - missing_cache_file = 7, - preprocessed_cache_hit = 8, - bad_compiler_arguments = 9, - called_for_link = 10, - files_in_cache = 11, - cache_size_kibibyte = 12, - obsolete_max_files = 13, - obsolete_max_size = 14, - unsupported_source_language = 15, - bad_output_file = 16, - no_input_file = 17, - multiple_source_files = 18, - autoconf_test = 19, - unsupported_compiler_option = 20, - output_to_stdout = 21, - direct_cache_hit = 22, - compiler_produced_no_output = 23, - compiler_produced_empty_output = 24, - error_hashing_extra_file = 25, - compiler_check_failed = 26, - could_not_use_precompiled_header = 27, - called_for_preprocessing = 28, - cleanups_performed = 29, - unsupported_code_directive = 30, - stats_zeroed_timestamp = 31, - could_not_use_modules = 32, - - END -}; diff -Nru ccache-4.2.1/src/Statistics.cpp ccache-4.5.1/src/Statistics.cpp --- ccache-4.2.1/src/Statistics.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Statistics.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,355 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Statistics.hpp" - -#include "AtomicFile.hpp" -#include "Config.hpp" -#include "Lockfile.hpp" -#include "Logging.hpp" -#include "Util.hpp" -#include "exceptions.hpp" -#include "fmtmacros.hpp" - -const unsigned FLAG_NOZERO = 1; // don't zero with the -z option -const unsigned FLAG_ALWAYS = 2; // always show, even if zero -const unsigned FLAG_NEVER = 4; // never show - -using nonstd::nullopt; -using nonstd::optional; - -// Returns a formatted version of a statistics value, or the empty string if the -// statistics line shouldn't be printed. -using FormatFunction = std::string (*)(uint64_t value); - -static std::string -format_size(uint64_t size) -{ - return FMT("{:>11}", Util::format_human_readable_size(size)); -} - -static std::string -format_size_times_1024(uint64_t size) -{ - return format_size(size * 1024); -} - -static std::string -format_timestamp(uint64_t timestamp) -{ - if (timestamp > 0) { - const auto tm = Util::localtime(timestamp); - char buffer[100] = "?"; - if (tm) { - strftime(buffer, sizeof(buffer), "%c", &*tm); - } - return std::string(" ") + buffer; - } else { - return {}; - } -} - -static double -hit_rate(const Counters& counters) -{ - const uint64_t direct = counters.get(Statistic::direct_cache_hit); - const uint64_t preprocessed = counters.get(Statistic::preprocessed_cache_hit); - const uint64_t hit = direct + preprocessed; - const uint64_t miss = counters.get(Statistic::cache_miss); - const uint64_t total = hit + miss; - return total > 0 ? (100.0 * hit) / total : 0.0; -} - -static void -for_each_level_1_and_2_stats_file( - const std::string& cache_dir, - const std::function function) -{ - for (size_t level_1 = 0; level_1 <= 0xF; ++level_1) { - function(FMT("{}/{:x}/stats", cache_dir, level_1)); - for (size_t level_2 = 0; level_2 <= 0xF; ++level_2) { - function(FMT("{}/{:x}/{:x}/stats", cache_dir, level_1, level_2)); - } - } -} - -static std::pair -collect_counters(const Config& config) -{ - Counters counters; - uint64_t zero_timestamp = 0; - time_t last_updated = 0; - - // Add up the stats in each directory. - for_each_level_1_and_2_stats_file( - config.cache_dir(), [&](const std::string& path) { - counters.set(Statistic::stats_zeroed_timestamp, 0); // Don't add - counters.increment(Statistics::read(path)); - zero_timestamp = std::max(counters.get(Statistic::stats_zeroed_timestamp), - zero_timestamp); - last_updated = std::max(last_updated, Stat::stat(path).mtime()); - }); - - counters.set(Statistic::stats_zeroed_timestamp, zero_timestamp); - return std::make_pair(counters, last_updated); -} - -namespace { - -struct StatisticsField -{ - StatisticsField(Statistic statistic_, - const char* id_, - const char* message_, - unsigned flags_ = 0, - FormatFunction format_ = nullptr) - : statistic(statistic_), - id(id_), - message(message_), - flags(flags_), - format(format_) - { - } - - const Statistic statistic; - const char* const id; // for --print-stats - const char* const message; // for --show-stats - const unsigned flags; // bitmask of FLAG_* values - const FormatFunction format; // nullptr -> use plain integer format -}; - -} // namespace - -#define STATISTICS_FIELD(id, ...) \ - { \ - Statistic::id, #id, __VA_ARGS__ \ - } - -// Statistics fields in display order. -const StatisticsField k_statistics_fields[] = { - STATISTICS_FIELD( - stats_zeroed_timestamp, "stats zeroed", FLAG_ALWAYS, format_timestamp), - STATISTICS_FIELD(direct_cache_hit, "cache hit (direct)", FLAG_ALWAYS), - STATISTICS_FIELD( - preprocessed_cache_hit, "cache hit (preprocessed)", FLAG_ALWAYS), - STATISTICS_FIELD(cache_miss, "cache miss", FLAG_ALWAYS), - STATISTICS_FIELD(called_for_link, "called for link"), - STATISTICS_FIELD(called_for_preprocessing, "called for preprocessing"), - STATISTICS_FIELD(multiple_source_files, "multiple source files"), - STATISTICS_FIELD(compiler_produced_stdout, "compiler produced stdout"), - STATISTICS_FIELD(compiler_produced_no_output, "compiler produced no output"), - STATISTICS_FIELD(compiler_produced_empty_output, - "compiler produced empty output"), - STATISTICS_FIELD(compile_failed, "compile failed"), - STATISTICS_FIELD(internal_error, "ccache internal error"), - STATISTICS_FIELD(preprocessor_error, "preprocessor error"), - STATISTICS_FIELD(could_not_use_precompiled_header, - "can't use precompiled header"), - STATISTICS_FIELD(could_not_use_modules, "can't use modules"), - STATISTICS_FIELD(could_not_find_compiler, "couldn't find the compiler"), - STATISTICS_FIELD(missing_cache_file, "cache file missing"), - STATISTICS_FIELD(bad_compiler_arguments, "bad compiler arguments"), - STATISTICS_FIELD(unsupported_source_language, "unsupported source language"), - STATISTICS_FIELD(compiler_check_failed, "compiler check failed"), - STATISTICS_FIELD(autoconf_test, "autoconf compile/link"), - STATISTICS_FIELD(unsupported_compiler_option, "unsupported compiler option"), - STATISTICS_FIELD(unsupported_code_directive, "unsupported code directive"), - STATISTICS_FIELD(output_to_stdout, "output to stdout"), - STATISTICS_FIELD(bad_output_file, "could not write to output file"), - STATISTICS_FIELD(no_input_file, "no input file"), - STATISTICS_FIELD(error_hashing_extra_file, "error hashing extra file"), - STATISTICS_FIELD(cleanups_performed, "cleanups performed", FLAG_ALWAYS), - STATISTICS_FIELD(files_in_cache, "files in cache", FLAG_NOZERO | FLAG_ALWAYS), - STATISTICS_FIELD(cache_size_kibibyte, - "cache size", - FLAG_NOZERO | FLAG_ALWAYS, - format_size_times_1024), - STATISTICS_FIELD(obsolete_max_files, "OBSOLETE", FLAG_NOZERO | FLAG_NEVER), - STATISTICS_FIELD(obsolete_max_size, "OBSOLETE", FLAG_NOZERO | FLAG_NEVER), - STATISTICS_FIELD(none, nullptr), -}; - -namespace Statistics { - -Counters -read(const std::string& path) -{ - Counters counters; - - std::string data; - try { - data = Util::read_file(path); - } catch (const Error&) { - // Ignore. - return counters; - } - - size_t i = 0; - const char* str = data.c_str(); - while (true) { - char* end; - const uint64_t value = std::strtoull(str, &end, 10); - if (end == str) { - break; - } - counters.set_raw(i, value); - ++i; - str = end; - } - - return counters; -} - -optional -update(const std::string& path, - std::function function) -{ - Lockfile lock(path); - if (!lock.acquired()) { - LOG("Failed to acquire lock for {}", path); - return nullopt; - } - - auto counters = Statistics::read(path); - function(counters); - - AtomicFile file(path, AtomicFile::Mode::text); - for (size_t i = 0; i < counters.size(); ++i) { - file.write(FMT("{}\n", counters.get_raw(i))); - } - try { - file.commit(); - } catch (const Error& e) { - // Make failure to write a stats file a soft error since it's not - // important enough to fail whole the process and also because it is - // called in the Context destructor. - LOG("Error: {}", e.what()); - } - - return counters; -} - -optional -get_result(const Counters& counters) -{ - for (const auto& field : k_statistics_fields) { - if (counters.get(field.statistic) != 0 && !(field.flags & FLAG_NOZERO)) { - return field.message; - } - } - return nullopt; -} - -void -zero_all_counters(const Config& config) -{ - const time_t timestamp = time(nullptr); - - for_each_level_1_and_2_stats_file( - config.cache_dir(), [=](const std::string& path) { - Statistics::update(path, [=](Counters& cs) { - for (size_t i = 0; k_statistics_fields[i].message; ++i) { - if (!(k_statistics_fields[i].flags & FLAG_NOZERO)) { - cs.set(k_statistics_fields[i].statistic, 0); - } - } - cs.set(Statistic::stats_zeroed_timestamp, timestamp); - }); - }); -} - -std::string -format_human_readable(const Config& config) -{ - Counters counters; - time_t last_updated; - std::tie(counters, last_updated) = collect_counters(config); - std::string result; - - result += FMT("{:36}{}\n", "cache directory", config.cache_dir()); - result += FMT("{:36}{}\n", "primary config", config.primary_config_path()); - result += FMT( - "{:36}{}\n", "secondary config (readonly)", config.secondary_config_path()); - if (last_updated > 0) { - const auto tm = Util::localtime(last_updated); - char timestamp[100] = "?"; - if (tm) { - strftime(timestamp, sizeof(timestamp), "%c", &*tm); - } - result += FMT("{:36}{}\n", "stats updated", timestamp); - } - - // ...and display them. - for (size_t i = 0; k_statistics_fields[i].message; i++) { - const Statistic statistic = k_statistics_fields[i].statistic; - - if (k_statistics_fields[i].flags & FLAG_NEVER) { - continue; - } - if (counters.get(statistic) == 0 - && !(k_statistics_fields[i].flags & FLAG_ALWAYS)) { - continue; - } - - const std::string value = - k_statistics_fields[i].format - ? k_statistics_fields[i].format(counters.get(statistic)) - : FMT("{:8}", counters.get(statistic)); - if (!value.empty()) { - result += FMT("{:32}{}\n", k_statistics_fields[i].message, value); - } - - if (statistic == Statistic::cache_miss) { - double percent = hit_rate(counters); - result += FMT("{:34}{:6.2f} %\n", "cache hit rate", percent); - } - } - - if (config.max_files() != 0) { - result += FMT("{:32}{:8}\n", "max files", config.max_files()); - } - if (config.max_size() != 0) { - result += - FMT("{:32}{}\n", "max cache size", format_size(config.max_size())); - } - - return result; -} - -std::string -format_machine_readable(const Config& config) -{ - Counters counters; - time_t last_updated; - std::tie(counters, last_updated) = collect_counters(config); - std::string result; - - result += FMT("stats_updated_timestamp\t{}\n", last_updated); - - for (size_t i = 0; k_statistics_fields[i].message; i++) { - if (!(k_statistics_fields[i].flags & FLAG_NEVER)) { - result += FMT("{}\t{}\n", - k_statistics_fields[i].id, - counters.get(k_statistics_fields[i].statistic)); - } - } - - return result; -} - -} // namespace Statistics diff -Nru ccache-4.2.1/src/Statistics.hpp ccache-4.5.1/src/Statistics.hpp --- ccache-4.2.1/src/Statistics.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/Statistics.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,58 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Counters.hpp" -#include "Statistic.hpp" // Any reasonable use of Statistics requires the Statistic enum. - -#include "third_party/nonstd/optional.hpp" - -#include -#include - -class Config; - -namespace Statistics { - -// Read counters from `path`. No lock is acquired. -Counters read(const std::string& path); - -// Acquire a lock, read counters from `path`, call `function` with the counters, -// write the counters to `path` and release the lock. Returns the resulting -// counters or nullopt on error (e.g. if the lock could not be acquired). -nonstd::optional update(const std::string& path, - std::function); - -// Return a human-readable string representing the final ccache result, or -// nullopt if there was no result. -nonstd::optional get_result(const Counters& counters); - -// Zero all statistics counters except those tracking cache size and number of -// files in the cache. -void zero_all_counters(const Config& config); - -// Format cache statistics in human-readable format. -std::string format_human_readable(const Config& config); - -// Format cache statistics in machine-readable format. -std::string format_machine_readable(const Config& config); - -} // namespace Statistics diff -Nru ccache-4.2.1/src/StdMakeUnique.hpp ccache-4.5.1/src/StdMakeUnique.hpp --- ccache-4.2.1/src/StdMakeUnique.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/StdMakeUnique.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,32 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -namespace std { - -#if __cplusplus < 201402L -template -inline unique_ptr -make_unique(TArgs&&... args) -{ - return unique_ptr(new T(std::forward(args)...)); -} -#endif - -} // namespace std diff -Nru ccache-4.2.1/src/storage/CMakeLists.txt ccache-4.5.1/src/storage/CMakeLists.txt --- ccache-4.2.1/src/storage/CMakeLists.txt 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,9 @@ +add_subdirectory(primary) +add_subdirectory(secondary) + +set( + sources + ${CMAKE_CURRENT_SOURCE_DIR}/Storage.cpp +) + +target_sources(ccache_framework PRIVATE ${sources}) diff -Nru ccache-4.2.1/src/storage/primary/CacheFile.cpp ccache-4.5.1/src/storage/primary/CacheFile.cpp --- ccache-4.2.1/src/storage/primary/CacheFile.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/CacheFile.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,47 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheFile.hpp" + +#include +#include +#include + +const Stat& +CacheFile::lstat() const +{ + if (!m_stat) { + m_stat = Stat::lstat(m_path); + } + + return *m_stat; +} + +CacheFile::Type +CacheFile::type() const +{ + if (util::ends_with(m_path, Manifest::k_file_suffix)) { + return Type::manifest; + } else if (util::ends_with(m_path, Result::k_file_suffix)) { + return Type::result; + } else if (util::ends_with(m_path, "W")) { + return Type::raw; + } else { + return Type::unknown; + } +} diff -Nru ccache-4.2.1/src/storage/primary/CacheFile.hpp ccache-4.5.1/src/storage/primary/CacheFile.hpp --- ccache-4.2.1/src/storage/primary/CacheFile.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/CacheFile.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,51 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include + +#include + +class CacheFile +{ +public: + enum class Type { result, manifest, raw, unknown }; + + explicit CacheFile(const std::string& path); + + const Stat& lstat() const; + const std::string& path() const; + Type type() const; + +private: + std::string m_path; + mutable nonstd::optional m_stat; +}; + +inline CacheFile::CacheFile(const std::string& path) : m_path(path) +{ +} + +inline const std::string& +CacheFile::path() const +{ + return m_path; +} diff -Nru ccache-4.2.1/src/storage/primary/CMakeLists.txt ccache-4.5.1/src/storage/primary/CMakeLists.txt --- ccache-4.2.1/src/storage/primary/CMakeLists.txt 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,12 @@ +set( + sources + ${CMAKE_CURRENT_SOURCE_DIR}/CacheFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_cleanup.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_compress.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/PrimaryStorage_statistics.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/StatsFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/util.cpp +) + +target_sources(ccache_framework PRIVATE ${sources}) diff -Nru ccache-4.2.1/src/storage/primary/PrimaryStorage_cleanup.cpp ccache-4.5.1/src/storage/primary/PrimaryStorage_cleanup.cpp --- ccache-4.2.1/src/storage/primary/PrimaryStorage_cleanup.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/PrimaryStorage_cleanup.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,282 @@ +// Copyright (C) 2002-2006 Andrew Tridgell +// Copyright (C) 2009-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "PrimaryStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef INODE_CACHE_SUPPORTED +# include +#endif + +#include + +using core::Statistic; + +namespace storage { +namespace primary { + +static void +delete_file(const std::string& path, + const uint64_t size, + uint64_t* cache_size, + uint64_t* files_in_cache) +{ + const bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure); + if (!deleted && errno != ENOENT && errno != ESTALE) { + LOG("Failed to unlink {} ({})", path, strerror(errno)); + } else if (cache_size && files_in_cache) { + // The counters are intentionally subtracted even if there was no file to + // delete since the final cache size calculation will be incorrect if they + // aren't. (This can happen when there are several parallel ongoing + // cleanups of the same directory.) + *cache_size -= size; + --*files_in_cache; + } +} + +static void +update_counters(const std::string& dir, + const uint64_t files_in_cache, + const uint64_t cache_size, + const bool cleanup_performed) +{ + const std::string stats_file = dir + "/stats"; + StatsFile(stats_file).update([=](auto& cs) { + if (cleanup_performed) { + cs.increment(Statistic::cleanups_performed); + } + cs.set(Statistic::files_in_cache, files_in_cache); + cs.set(Statistic::cache_size_kibibyte, cache_size / 1024); + }); +} + +void +PrimaryStorage::evict(const ProgressReceiver& progress_receiver, + nonstd::optional max_age, + nonstd::optional namespace_) +{ + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const std::string& subdir, + const ProgressReceiver& sub_progress_receiver) { + clean_dir(subdir, 0, 0, max_age, namespace_, sub_progress_receiver); + }, + progress_receiver); +} + +// Clean up one cache subdirectory. +void +PrimaryStorage::clean_dir(const std::string& subdir, + const uint64_t max_size, + const uint64_t max_files, + const nonstd::optional max_age, + const nonstd::optional namespace_, + const ProgressReceiver& progress_receiver) +{ + LOG("Cleaning up cache directory {}", subdir); + + std::vector files = get_level_1_files( + subdir, [&](double progress) { progress_receiver(progress / 3); }); + + uint64_t cache_size = 0; + uint64_t files_in_cache = 0; + time_t current_time = time(nullptr); + std::unordered_map /*associated_raw_files*/> + raw_files_map; + + for (size_t i = 0; i < files.size(); + ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) { + const auto& file = files[i]; + + if (!file.lstat().is_regular()) { + // Not a file or missing file. + continue; + } + + // Delete any tmp files older than 1 hour right away. + if (file.lstat().mtime() + 3600 < current_time + && Util::base_name(file.path()).find(".tmp.") != std::string::npos) { + Util::unlink_tmp(file.path()); + continue; + } + + if (namespace_ && file.type() == CacheFile::Type::raw) { + const auto result_filename = + FMT("{}R", file.path().substr(0, file.path().length() - 2)); + raw_files_map[result_filename].push_back(file.path()); + } + + cache_size += file.lstat().size_on_disk(); + files_in_cache += 1; + } + + // Sort according to modification time, oldest first. + std::sort(files.begin(), files.end(), [](const auto& f1, const auto& f2) { + const auto ts_1 = f1.lstat().mtim(); + const auto ts_2 = f2.lstat().mtim(); + const auto ns_1 = 1'000'000'000ULL * ts_1.tv_sec + ts_1.tv_nsec; + const auto ns_2 = 1'000'000'000ULL * ts_2.tv_sec + ts_2.tv_nsec; + return ns_1 < ns_2; + }); + + LOG("Before cleanup: {:.0f} KiB, {:.0f} files", + static_cast(cache_size) / 1024, + static_cast(files_in_cache)); + + bool cleaned = false; + for (size_t i = 0; i < files.size(); + ++i, progress_receiver(2.0 / 3 + 1.0 * i / files.size() / 3)) { + const auto& file = files[i]; + + if (!file.lstat() || file.lstat().is_directory()) { + continue; + } + + if ((max_size == 0 || cache_size <= max_size) + && (max_files == 0 || files_in_cache <= max_files) + && (!max_age + || file.lstat().mtime() + > (current_time - static_cast(*max_age))) + && (!namespace_ || max_age)) { + break; + } + + if (namespace_) { + try { + File file_stream(file.path(), "rb"); + core::FileReader file_reader(*file_stream); + core::CacheEntryReader reader(file_reader); + if (reader.header().namespace_ != *namespace_) { + continue; + } + } catch (core::Error&) { + // Failed to read header: ignore. + continue; + } + + // For namespace eviction we need to remove raw files based on result + // filename since they don't have a header. + if (file.type() == CacheFile::Type::result) { + const auto entry = raw_files_map.find(file.path()); + if (entry != raw_files_map.end()) { + for (const auto& raw_file : entry->second) { + delete_file(raw_file, + Stat::lstat(raw_file).size_on_disk(), + &cache_size, + &files_in_cache); + } + } + } + } + + if (util::ends_with(file.path(), ".stderr")) { + // In order to be nice to legacy ccache versions, make sure that the .o + // file is deleted before .stderr, because if the ccache process gets + // killed after deleting the .stderr but before deleting the .o, the + // cached result will be inconsistent. (.stderr is the only file that is + // optional for legacy ccache versions; any other file missing from the + // cache will be detected.) + std::string o_file = file.path().substr(0, file.path().size() - 6) + "o"; + + // Don't subtract this extra deletion from the cache size; that + // bookkeeping will be done when the loop reaches the .o file. If the + // loop doesn't reach the .o file since the target limits have been + // reached, the bookkeeping won't happen, but that small counter + // discrepancy won't do much harm and it will correct itself in the next + // cleanup. + delete_file(o_file, 0, nullptr, nullptr); + } + + delete_file( + file.path(), file.lstat().size_on_disk(), &cache_size, &files_in_cache); + cleaned = true; + } + + LOG("After cleanup: {:.0f} KiB, {:.0f} files", + static_cast(cache_size) / 1024, + static_cast(files_in_cache)); + + if (cleaned) { + LOG("Cleaned up cache directory {}", subdir); + } + + update_counters(subdir, files_in_cache, cache_size, cleaned); +} + +// Clean up all cache subdirectories. +void +PrimaryStorage::clean_all(const ProgressReceiver& progress_receiver) +{ + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const std::string& subdir, + const ProgressReceiver& sub_progress_receiver) { + clean_dir(subdir, + m_config.max_size() / 16, + m_config.max_files() / 16, + nonstd::nullopt, + nonstd::nullopt, + sub_progress_receiver); + }, + progress_receiver); +} + +// Wipe one cache subdirectory. +static void +wipe_dir(const std::string& subdir, const ProgressReceiver& progress_receiver) +{ + LOG("Clearing out cache directory {}", subdir); + + const std::vector files = get_level_1_files( + subdir, [&](double progress) { progress_receiver(progress / 2); }); + + for (size_t i = 0; i < files.size(); ++i) { + Util::unlink_safe(files[i].path()); + progress_receiver(0.5 + 0.5 * i / files.size()); + } + + const bool cleared = !files.empty(); + if (cleared) { + LOG("Cleared out cache directory {}", subdir); + } + update_counters(subdir, 0, 0, cleared); +} + +// Wipe all cached files in all subdirectories. +void +PrimaryStorage::wipe_all(const ProgressReceiver& progress_receiver) +{ + for_each_level_1_subdir(m_config.cache_dir(), wipe_dir, progress_receiver); +} + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/PrimaryStorage_compress.cpp ccache-4.5.1/src/storage/primary/PrimaryStorage_compress.cpp --- ccache-4.2.1/src/storage/primary/PrimaryStorage_compress.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/PrimaryStorage_compress.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,338 @@ +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "PrimaryStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +#include +#include +#include + +namespace storage { +namespace primary { + +namespace { + +class RecompressionStatistics +{ +public: + void update(uint64_t content_size, + uint64_t old_size, + uint64_t new_size, + uint64_t incompressible_size); + uint64_t content_size() const; + uint64_t old_size() const; + uint64_t new_size() const; + uint64_t incompressible_size() const; + +private: + mutable std::mutex m_mutex; + uint64_t m_content_size = 0; + uint64_t m_old_size = 0; + uint64_t m_new_size = 0; + uint64_t m_incompressible_size = 0; +}; + +void +RecompressionStatistics::update(const uint64_t content_size, + const uint64_t old_size, + const uint64_t new_size, + const uint64_t incompressible_size) +{ + std::unique_lock lock(m_mutex); + m_incompressible_size += incompressible_size; + m_content_size += content_size; + m_old_size += old_size; + m_new_size += new_size; +} + +uint64_t +RecompressionStatistics::content_size() const +{ + std::unique_lock lock(m_mutex); + return m_content_size; +} + +uint64_t +RecompressionStatistics::old_size() const +{ + std::unique_lock lock(m_mutex); + return m_old_size; +} + +uint64_t +RecompressionStatistics::new_size() const +{ + std::unique_lock lock(m_mutex); + return m_new_size; +} + +uint64_t +RecompressionStatistics::incompressible_size() const +{ + std::unique_lock lock(m_mutex); + return m_incompressible_size; +} + +} // namespace + +static File +open_file(const std::string& path, const char* const mode) +{ + File f(path, mode); + if (!f) { + throw core::Error( + "failed to open {} for reading: {}", path, strerror(errno)); + } + return f; +} + +static std::unique_ptr +create_reader(const CacheFile& cache_file, core::Reader& reader) +{ + if (cache_file.type() == CacheFile::Type::unknown) { + throw core::Error("unknown file type for {}", cache_file.path()); + } + + return std::make_unique(reader); +} + +static std::unique_ptr +create_writer(core::Writer& writer, const core::CacheEntryHeader& header) +{ + return std::make_unique(writer, header); +} + +static void +recompress_file(RecompressionStatistics& statistics, + const std::string& stats_file, + const CacheFile& cache_file, + const nonstd::optional level) +{ + auto file = open_file(cache_file.path(), "rb"); + core::FileReader file_reader(file.get()); + auto reader = create_reader(cache_file, file_reader); + + const auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log); + const uint64_t content_size = reader->header().entry_size; + const int8_t wanted_level = + level + ? (*level == 0 ? compression::ZstdCompressor::default_compression_level + : *level) + : 0; + + if (reader->header().compression_level == wanted_level) { + statistics.update(content_size, old_stat.size(), old_stat.size(), 0); + return; + } + + LOG("Recompressing {} to {}", + cache_file.path(), + level ? FMT("level {}", wanted_level) : "uncompressed"); + AtomicFile atomic_new_file(cache_file.path(), AtomicFile::Mode::binary); + core::FileWriter file_writer(atomic_new_file.stream()); + auto header = reader->header(); + header.compression_type = + level ? compression::Type::zstd : compression::Type::none; + header.compression_level = wanted_level; + auto writer = create_writer(file_writer, header); + + char buffer[CCACHE_READ_BUFFER_SIZE]; + size_t bytes_left = reader->header().payload_size(); + while (bytes_left > 0) { + size_t bytes_to_read = std::min(bytes_left, sizeof(buffer)); + reader->read(buffer, bytes_to_read); + writer->write(buffer, bytes_to_read); + bytes_left -= bytes_to_read; + } + reader->finalize(); + writer->finalize(); + + file.close(); + + atomic_new_file.commit(); + const auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log); + + StatsFile(stats_file).update([=](auto& cs) { + cs.increment(core::Statistic::cache_size_kibibyte, + Util::size_change_kibibyte(old_stat, new_stat)); + }); + + statistics.update(content_size, old_stat.size(), new_stat.size(), 0); + + LOG("Recompression of {} done", cache_file.path()); +} + +CompressionStatistics +PrimaryStorage::get_compression_statistics( + const ProgressReceiver& progress_receiver) const +{ + CompressionStatistics cs{}; + + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const auto& subdir, const auto& sub_progress_receiver) { + const std::vector files = get_level_1_files( + subdir, [&](double progress) { sub_progress_receiver(progress / 2); }); + + for (size_t i = 0; i < files.size(); ++i) { + const auto& cache_file = files[i]; + cs.on_disk_size += cache_file.lstat().size_on_disk(); + + try { + auto file = open_file(cache_file.path(), "rb"); + core::FileReader file_reader(file.get()); + auto reader = create_reader(cache_file, file_reader); + cs.compr_size += cache_file.lstat().size(); + cs.content_size += reader->header().entry_size; + } catch (core::Error&) { + cs.incompr_size += cache_file.lstat().size(); + } + + sub_progress_receiver(1.0 / 2 + 1.0 * i / files.size() / 2); + } + }, + progress_receiver); + + return cs; +} + +void +PrimaryStorage::recompress(const nonstd::optional level, + const ProgressReceiver& progress_receiver) +{ + const size_t threads = std::thread::hardware_concurrency(); + const size_t read_ahead = 2 * threads; + ThreadPool thread_pool(threads, read_ahead); + RecompressionStatistics statistics; + + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const auto& subdir, const auto& sub_progress_receiver) { + std::vector files = + get_level_1_files(subdir, [&](double progress) { + sub_progress_receiver(0.1 * progress); + }); + + auto stats_file = subdir + "/stats"; + + for (size_t i = 0; i < files.size(); ++i) { + const auto& file = files[i]; + + if (file.type() != CacheFile::Type::unknown) { + thread_pool.enqueue([&statistics, stats_file, file, level] { + try { + recompress_file(statistics, stats_file, file, level); + } catch (core::Error&) { + // Ignore for now. + } + }); + } else { + statistics.update(0, 0, 0, file.lstat().size()); + } + + sub_progress_receiver(0.1 + 0.9 * i / files.size()); + } + + if (util::ends_with(subdir, "f")) { + // Wait here instead of after for_each_level_1_subdir to avoid + // updating the progress bar to 100% before all work is done. + thread_pool.shut_down(); + } + }, + progress_receiver); + + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n\n"); + } + + const double old_ratio = + statistics.old_size() > 0 + ? static_cast(statistics.content_size()) / statistics.old_size() + : 0.0; + const double old_savings = + old_ratio > 0.0 ? 100.0 - (100.0 / old_ratio) : 0.0; + const double new_ratio = + statistics.new_size() > 0 + ? static_cast(statistics.content_size()) / statistics.new_size() + : 0.0; + const double new_savings = + new_ratio > 0.0 ? 100.0 - (100.0 / new_ratio) : 0.0; + const int64_t size_difference = static_cast(statistics.new_size()) + - static_cast(statistics.old_size()); + + const std::string old_compr_size_str = + Util::format_human_readable_size(statistics.old_size()); + const std::string new_compr_size_str = + Util::format_human_readable_size(statistics.new_size()); + const std::string content_size_str = + Util::format_human_readable_size(statistics.content_size()); + const std::string incompr_size_str = + Util::format_human_readable_size(statistics.incompressible_size()); + const std::string size_difference_str = + FMT("{}{}", + size_difference < 0 ? "-" : (size_difference > 0 ? "+" : " "), + Util::format_human_readable_size( + size_difference < 0 ? -size_difference : size_difference)); + + PRINT(stdout, "Original data: {:>8s}\n", content_size_str); + PRINT(stdout, + "Old compressed data: {:>8s} ({:.1f}% of original size)\n", + old_compr_size_str, + 100.0 - old_savings); + PRINT(stdout, + " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", + old_ratio, + old_savings); + PRINT(stdout, + "New compressed data: {:>8s} ({:.1f}% of original size)\n", + new_compr_size_str, + 100.0 - new_savings); + PRINT(stdout, + " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", + new_ratio, + new_savings); + PRINT(stdout, "Size change: {:>9s}\n", size_difference_str); +} + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/PrimaryStorage.cpp ccache-4.5.1/src/storage/primary/PrimaryStorage.cpp --- ccache-4.2.1/src/storage/primary/PrimaryStorage.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/PrimaryStorage.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,409 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "PrimaryStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +using core::Statistic; + +namespace storage { +namespace primary { + +// How often (in seconds) to scan $CCACHE_DIR/tmp for left-over temporary +// files. +const int k_tempdir_cleanup_interval = 2 * 24 * 60 * 60; // 2 days + +// Maximum files per cache directory. This constant is somewhat arbitrarily +// chosen to be large enough to avoid unnecessary cache levels but small enough +// not to make esoteric file systems (with bad performance for large +// directories) too slow. It could be made configurable, but hopefully there +// will be no need to do that. +const uint64_t k_max_cache_files_per_directory = 2000; + +// Minimum number of cache levels ($CCACHE_DIR/1/2/stored_file). +const uint8_t k_min_cache_levels = 2; + +// Maximum number of cache levels ($CCACHE_DIR/1/2/3/stored_file). +// +// On a cache miss, (k_max_cache_levels - k_min_cache_levels + 1) cache lookups +// (i.e. stat system calls) will be performed for a cache entry. +// +// An assumption made here is that if a cache is so large that it holds more +// than 16^4 * k_max_cache_files_per_directory files then we can assume that the +// file system is sane enough to handle more than +// k_max_cache_files_per_directory. +const uint8_t k_max_cache_levels = 4; + +static std::string +suffix_from_type(const core::CacheEntryType type) +{ + switch (type) { + case core::CacheEntryType::manifest: + return "M"; + + case core::CacheEntryType::result: + return "R"; + } + + ASSERT(false); +} + +static uint8_t +calculate_wanted_cache_level(const uint64_t files_in_level_1) +{ + uint64_t files_per_directory = files_in_level_1 / 16; + for (uint8_t i = k_min_cache_levels; i <= k_max_cache_levels; ++i) { + if (files_per_directory < k_max_cache_files_per_directory) { + return i; + } + files_per_directory /= 16; + } + return k_max_cache_levels; +} + +PrimaryStorage::PrimaryStorage(const Config& config) : m_config(config) +{ +} + +void +PrimaryStorage::initialize() +{ + MTR_SCOPE("primary_storage", "clean_internal_tempdir"); + + if (m_config.temporary_dir() == m_config.cache_dir() + "/tmp") { + clean_internal_tempdir(); + } +} + +void +PrimaryStorage::finalize() +{ + if (!m_config.stats()) { + return; + } + + if (m_manifest_key) { + // A manifest entry was written. + ASSERT(!m_manifest_path.empty()); + update_stats_and_maybe_move_cache_file(*m_manifest_key, + m_manifest_path, + m_manifest_counter_updates, + core::CacheEntryType::manifest); + } + + if (!m_result_key) { + // No result entry was written, so we just choose one of the stats files in + // the 256 level 2 directories. + + ASSERT(m_result_counter_updates.get(Statistic::cache_size_kibibyte) == 0); + ASSERT(m_result_counter_updates.get(Statistic::files_in_cache) == 0); + + const auto bucket = getpid() % 256; + const auto stats_file = + FMT("{}/{:x}/{:x}/stats", m_config.cache_dir(), bucket / 16, bucket % 16); + StatsFile(stats_file).update([&](auto& cs) { + cs.increment(m_result_counter_updates); + }); + return; + } + + ASSERT(!m_result_path.empty()); + + const auto counters = + update_stats_and_maybe_move_cache_file(*m_result_key, + m_result_path, + m_result_counter_updates, + core::CacheEntryType::result); + if (!counters) { + return; + } + + const auto subdir = + FMT("{}/{:x}", m_config.cache_dir(), m_result_key->bytes()[0] >> 4); + bool need_cleanup = false; + + if (m_config.max_files() != 0 + && counters->get(Statistic::files_in_cache) > m_config.max_files() / 16) { + LOG("Need to clean up {} since it holds {} files (limit: {} files)", + subdir, + counters->get(Statistic::files_in_cache), + m_config.max_files() / 16); + need_cleanup = true; + } + if (m_config.max_size() != 0 + && counters->get(Statistic::cache_size_kibibyte) + > m_config.max_size() / 1024 / 16) { + LOG("Need to clean up {} since it holds {} KiB (limit: {} KiB)", + subdir, + counters->get(Statistic::cache_size_kibibyte), + m_config.max_size() / 1024 / 16); + need_cleanup = true; + } + + if (need_cleanup) { + const double factor = m_config.limit_multiple() / 16; + const uint64_t max_size = round(m_config.max_size() * factor); + const uint32_t max_files = round(m_config.max_files() * factor); + clean_dir(subdir, + max_size, + max_files, + nonstd::nullopt, + nonstd::nullopt, + [](double /*progress*/) {}); + } +} + +nonstd::optional +PrimaryStorage::get(const Digest& key, const core::CacheEntryType type) const +{ + MTR_SCOPE("primary_storage", "get"); + + const auto cache_file = look_up_cache_file(key, type); + if (!cache_file.stat) { + LOG("No {} in primary storage", key.to_string()); + return nonstd::nullopt; + } + + LOG( + "Retrieved {} from primary storage ({})", key.to_string(), cache_file.path); + + // Update modification timestamp to save file from LRU cleanup. + Util::update_mtime(cache_file.path); + return cache_file.path; +} + +nonstd::optional +PrimaryStorage::put(const Digest& key, + const core::CacheEntryType type, + const storage::EntryWriter& entry_writer) +{ + MTR_SCOPE("primary_storage", "put"); + + const auto cache_file = look_up_cache_file(key, type); + switch (type) { + case core::CacheEntryType::manifest: + m_manifest_key = key; + m_manifest_path = cache_file.path; + break; + + case core::CacheEntryType::result: + m_result_key = key; + m_result_path = cache_file.path; + break; + } + + if (!entry_writer(cache_file.path)) { + LOG("Did not store {} in primary storage", key.to_string()); + return nonstd::nullopt; + } + + const auto new_stat = Stat::stat(cache_file.path, Stat::OnError::log); + if (!new_stat) { + LOG("Failed to stat {}: {}", cache_file.path, strerror(errno)); + return nonstd::nullopt; + } + + LOG("Stored {} in primary storage ({})", key.to_string(), cache_file.path); + + auto& counter_updates = (type == core::CacheEntryType::manifest) + ? m_manifest_counter_updates + : m_result_counter_updates; + counter_updates.increment( + Statistic::cache_size_kibibyte, + Util::size_change_kibibyte(cache_file.stat, new_stat)); + counter_updates.increment(Statistic::files_in_cache, cache_file.stat ? 0 : 1); + + // Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can + // be done almost anywhere, but we might as well do it near the end as we save + // the stat call if we exit early. + util::create_cachedir_tag( + FMT("{}/{}", m_config.cache_dir(), key.to_string()[0])); + + return cache_file.path; +} + +void +PrimaryStorage::remove(const Digest& key, const core::CacheEntryType type) +{ + MTR_SCOPE("primary_storage", "remove"); + + const auto cache_file = look_up_cache_file(key, type); + if (cache_file.stat) { + Util::unlink_safe(cache_file.path); + LOG( + "Removed {} from primary storage ({})", key.to_string(), cache_file.path); + } else { + LOG("No {} to remove from primary storage", key.to_string()); + } +} + +void +PrimaryStorage::increment_statistic(const Statistic statistic, + const int64_t value) +{ + m_result_counter_updates.increment(statistic, value); +} + +void +PrimaryStorage::increment_statistics(const core::StatisticsCounters& statistics) +{ + m_result_counter_updates.increment(statistics); +} + +// Private methods + +PrimaryStorage::LookUpCacheFileResult +PrimaryStorage::look_up_cache_file(const Digest& key, + const core::CacheEntryType type) const +{ + const auto key_string = FMT("{}{}", key.to_string(), suffix_from_type(type)); + + for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; + ++level) { + const auto path = get_path_in_cache(level, key_string); + const auto stat = Stat::stat(path); + if (stat) { + return {path, stat, level}; + } + } + + const auto shallowest_path = + get_path_in_cache(k_min_cache_levels, key_string); + return {shallowest_path, Stat(), k_min_cache_levels}; +} + +void +PrimaryStorage::clean_internal_tempdir() +{ + const time_t now = time(nullptr); + const auto dir_st = Stat::stat(m_config.cache_dir(), Stat::OnError::log); + if (!dir_st || dir_st.mtime() + k_tempdir_cleanup_interval >= now) { + // No cleanup needed. + return; + } + + Util::update_mtime(m_config.cache_dir()); + + const std::string& temp_dir = m_config.temporary_dir(); + if (!Stat::lstat(temp_dir)) { + return; + } + + Util::traverse(temp_dir, [now](const std::string& path, bool is_dir) { + if (is_dir) { + return; + } + const auto st = Stat::lstat(path, Stat::OnError::log); + if (st && st.mtime() + k_tempdir_cleanup_interval < now) { + Util::unlink_tmp(path); + } + }); +} + +nonstd::optional +PrimaryStorage::update_stats_and_maybe_move_cache_file( + const Digest& key, + const std::string& current_path, + const core::StatisticsCounters& counter_updates, + const core::CacheEntryType type) +{ + if (counter_updates.all_zero()) { + return nonstd::nullopt; + } + + // Use stats file in the level one subdirectory for cache bookkeeping counters + // since cleanup is performed on level one. Use stats file in the level two + // subdirectory for other counters to reduce lock contention. + const bool use_stats_on_level_1 = + counter_updates.get(Statistic::cache_size_kibibyte) != 0 + || counter_updates.get(Statistic::files_in_cache) != 0; + std::string level_string = FMT("{:x}", key.bytes()[0] >> 4); + if (!use_stats_on_level_1) { + level_string += FMT("/{:x}", key.bytes()[0] & 0xF); + } + + const auto stats_file = + FMT("{}/{}/stats", m_config.cache_dir(), level_string); + const auto counters = + StatsFile(stats_file).update([&counter_updates](auto& cs) { + cs.increment(counter_updates); + }); + if (!counters) { + return nonstd::nullopt; + } + + if (use_stats_on_level_1) { + // Only consider moving the cache file to another level when we have read + // the level 1 stats file since it's only then we know the proper + // files_in_cache value. + const auto wanted_level = + calculate_wanted_cache_level(counters->get(Statistic::files_in_cache)); + const auto wanted_path = + get_path_in_cache(wanted_level, key.to_string() + suffix_from_type(type)); + if (current_path != wanted_path) { + Util::ensure_dir_exists(Util::dir_name(wanted_path)); + LOG("Moving {} to {}", current_path, wanted_path); + try { + Util::rename(current_path, wanted_path); + } catch (const core::Error&) { + // Two ccache processes may move the file at the same time, so failure + // to rename is OK. + } + } + } + return counters; +} + +std::string +PrimaryStorage::get_path_in_cache(const uint8_t level, + const nonstd::string_view name) const +{ + ASSERT(level >= 1 && level <= 8); + ASSERT(name.length() >= level); + + std::string path(m_config.cache_dir()); + path.reserve(path.size() + level * 2 + 1 + name.length() - level); + + for (uint8_t i = 0; i < level; ++i) { + path.push_back('/'); + path.push_back(name.at(i)); + } + + path.push_back('/'); + const nonstd::string_view name_remaining = name.substr(level); + path.append(name_remaining.data(), name_remaining.length()); + + return path; +} + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/PrimaryStorage.hpp ccache-4.5.1/src/storage/primary/PrimaryStorage.hpp --- ccache-4.2.1/src/storage/primary/PrimaryStorage.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/PrimaryStorage.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,156 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +class Config; + +namespace storage { +namespace primary { + +struct CompressionStatistics +{ + uint64_t compr_size; + uint64_t content_size; + uint64_t incompr_size; + uint64_t on_disk_size; +}; + +class PrimaryStorage +{ +public: + PrimaryStorage(const Config& config); + + void initialize(); + void finalize(); + + // --- Cache entry handling --- + + // Returns a path to a file containing the value. + nonstd::optional get(const Digest& key, + core::CacheEntryType type) const; + + nonstd::optional put(const Digest& key, + core::CacheEntryType type, + const storage::EntryWriter& entry_writer); + + void remove(const Digest& key, core::CacheEntryType type); + + // --- Statistics --- + + void increment_statistic(core::Statistic statistic, int64_t value = 1); + void increment_statistics(const core::StatisticsCounters& statistics); + + const core::StatisticsCounters& get_statistics_updates() const; + + // Zero all statistics counters except those tracking cache size and number of + // files in the cache. + void zero_all_statistics(); + + // Get statistics and last time of update for the whole primary storage cache. + std::pair get_all_statistics() const; + + // --- Cleanup --- + + void evict(const ProgressReceiver& progress_receiver, + nonstd::optional max_age, + nonstd::optional namespace_); + + void clean_all(const ProgressReceiver& progress_receiver); + + void wipe_all(const ProgressReceiver& progress_receiver); + + // --- Compression --- + + CompressionStatistics + get_compression_statistics(const ProgressReceiver& progress_receiver) const; + + void recompress(nonstd::optional level, + const ProgressReceiver& progress_receiver); + +private: + const Config& m_config; + + // Main statistics updates (result statistics and size/count change for result + // file) which get written into the statistics file belonging to the result + // file. + core::StatisticsCounters m_result_counter_updates; + + // Statistics updates (only for manifest size/count change) which get written + // into the statistics file belonging to the manifest. + core::StatisticsCounters m_manifest_counter_updates; + + // The manifest and result keys and paths are stored by put() so that + // finalize() can use them to move the files in place. + nonstd::optional m_manifest_key; + nonstd::optional m_result_key; + std::string m_manifest_path; + std::string m_result_path; + + struct LookUpCacheFileResult + { + std::string path; + Stat stat; + uint8_t level; + }; + + LookUpCacheFileResult look_up_cache_file(const Digest& key, + core::CacheEntryType type) const; + + void clean_internal_tempdir(); + + nonstd::optional + update_stats_and_maybe_move_cache_file( + const Digest& key, + const std::string& current_path, + const core::StatisticsCounters& counter_updates, + core::CacheEntryType type); + + // Join the cache directory, a '/' and `name` into a single path and return + // it. Additionally, `level` single-character, '/'-separated subpaths are + // split from the beginning of `name` before joining them all. + std::string get_path_in_cache(uint8_t level, nonstd::string_view name) const; + + static void clean_dir(const std::string& subdir, + uint64_t max_size, + uint64_t max_files, + nonstd::optional max_age, + nonstd::optional namespace_, + const ProgressReceiver& progress_receiver); +}; + +// --- Inline implementations --- + +inline const core::StatisticsCounters& +PrimaryStorage::get_statistics_updates() const +{ + return m_result_counter_updates; +} + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/PrimaryStorage_statistics.cpp ccache-4.5.1/src/storage/primary/PrimaryStorage_statistics.cpp --- ccache-4.2.1/src/storage/primary/PrimaryStorage_statistics.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/PrimaryStorage_statistics.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,86 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "PrimaryStorage.hpp" + +#include +#include +#include +#include + +#include + +namespace storage { +namespace primary { + +static void +for_each_level_1_and_2_stats_file( + const std::string& cache_dir, + const std::function function) +{ + for (size_t level_1 = 0; level_1 <= 0xF; ++level_1) { + function(FMT("{}/{:x}/stats", cache_dir, level_1)); + for (size_t level_2 = 0; level_2 <= 0xF; ++level_2) { + function(FMT("{}/{:x}/{:x}/stats", cache_dir, level_1, level_2)); + } + } +} + +// Zero all statistics counters except those tracking cache size and number of +// files in the cache. +void +PrimaryStorage::zero_all_statistics() +{ + const time_t timestamp = time(nullptr); + const auto zeroable_fields = core::Statistics::get_zeroable_fields(); + + for_each_level_1_and_2_stats_file( + m_config.cache_dir(), [=](const std::string& path) { + StatsFile(path).update([=](auto& cs) { + for (const auto statistic : zeroable_fields) { + cs.set(statistic, 0); + } + cs.set(core::Statistic::stats_zeroed_timestamp, timestamp); + }); + }); +} + +// Get statistics and last time of update for the whole primary storage cache. +std::pair +PrimaryStorage::get_all_statistics() const +{ + core::StatisticsCounters counters; + uint64_t zero_timestamp = 0; + time_t last_updated = 0; + + // Add up the stats in each directory. + for_each_level_1_and_2_stats_file( + m_config.cache_dir(), [&](const auto& path) { + counters.set(core::Statistic::stats_zeroed_timestamp, 0); // Don't add + counters.increment(StatsFile(path).read()); + zero_timestamp = std::max( + counters.get(core::Statistic::stats_zeroed_timestamp), zero_timestamp); + last_updated = std::max(last_updated, Stat::stat(path).mtime()); + }); + + counters.set(core::Statistic::stats_zeroed_timestamp, zero_timestamp); + return std::make_pair(counters, last_updated); +} + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/StatsFile.cpp ccache-4.5.1/src/storage/primary/StatsFile.cpp --- ccache-4.2.1/src/storage/primary/StatsFile.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/StatsFile.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,94 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatsFile.hpp" + +#include +#include +#include +#include +#include +#include + +namespace storage { +namespace primary { + +StatsFile::StatsFile(const std::string& path) : m_path(path) +{ +} + +core::StatisticsCounters +StatsFile::read() const +{ + core::StatisticsCounters counters; + + std::string data; + try { + data = Util::read_file(m_path); + } catch (const core::Error&) { + // Ignore. + return counters; + } + + size_t i = 0; + const char* str = data.c_str(); + while (true) { + char* end; + const uint64_t value = std::strtoull(str, &end, 10); + if (end == str) { + break; + } + counters.set_raw(i, value); + ++i; + str = end; + } + + return counters; +} + +nonstd::optional +StatsFile::update( + std::function function) const +{ + Lockfile lock(m_path); + if (!lock.acquired()) { + LOG("Failed to acquire lock for {}", m_path); + return nonstd::nullopt; + } + + auto counters = read(); + function(counters); + + AtomicFile file(m_path, AtomicFile::Mode::text); + for (size_t i = 0; i < counters.size(); ++i) { + file.write(FMT("{}\n", counters.get_raw(i))); + } + try { + file.commit(); + } catch (const core::Error& e) { + // Make failure to write a stats file a soft error since it's not important + // enough to fail whole the process and also because it is called in the + // Context destructor. + LOG("Error: {}", e.what()); + } + + return counters; +} + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/StatsFile.hpp ccache-4.5.1/src/storage/primary/StatsFile.hpp --- ccache-4.2.1/src/storage/primary/StatsFile.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/StatsFile.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,51 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include + +#include +#include + +namespace storage { +namespace primary { + +class StatsFile +{ +public: + StatsFile(const std::string& path); + + // Read counters. No lock is acquired. If the file doesn't exist all returned + // counters will be zero. + core::StatisticsCounters read() const; + + // Acquire a lock, read counters, call `function` with the counters, write the + // counters and release the lock. Returns the resulting counters or nullopt on + // error (e.g. if the lock could not be acquired). + nonstd::optional + update(std::function) const; + +private: + const std::string m_path; +}; + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/util.cpp ccache-4.5.1/src/storage/primary/util.cpp --- ccache-4.2.1/src/storage/primary/util.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/util.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,75 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "util.hpp" + +#include +#include + +namespace storage { +namespace primary { + +void +for_each_level_1_subdir(const std::string& cache_dir, + const SubdirVisitor& visitor, + const ProgressReceiver& progress_receiver) +{ + for (int i = 0; i <= 0xF; i++) { + double progress = 1.0 * i / 16; + progress_receiver(progress); + std::string subdir_path = FMT("{}/{:x}", cache_dir, i); + visitor(subdir_path, [&](double inner_progress) { + progress_receiver(progress + inner_progress / 16); + }); + } + progress_receiver(1.0); +} + +std::vector +get_level_1_files(const std::string& dir, + const ProgressReceiver& progress_receiver) +{ + std::vector files; + + if (!Stat::stat(dir)) { + return files; + } + + size_t level_2_directories = 0; + + Util::traverse(dir, [&](const std::string& path, bool is_dir) { + auto name = Util::base_name(path); + if (name == "CACHEDIR.TAG" || name == "stats" || name.starts_with(".nfs")) { + return; + } + + if (!is_dir) { + files.emplace_back(path); + } else if (path != dir + && path.find('/', dir.size() + 1) == std::string::npos) { + ++level_2_directories; + progress_receiver(level_2_directories / 16.0); + } + }); + + progress_receiver(1.0); + return files; +} + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/primary/util.hpp ccache-4.5.1/src/storage/primary/util.hpp --- ccache-4.2.1/src/storage/primary/util.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/primary/util.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,64 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include +#include + +namespace storage { +namespace primary { + +using ProgressReceiver = std::function; +using SubdirVisitor = std::function; + +// Call a function for each subdir (0-9a-f) in the cache. +// +// Parameters: +// - cache_dir: Path to the cache directory. +// - visitor: Function to call with directory path and progress_receiver as +// arguments. +// - progress_receiver: Function that will be called for progress updates. +void for_each_level_1_subdir(const std::string& cache_dir, + const SubdirVisitor& visitor, + const ProgressReceiver& progress_receiver); + +// Get a list of files in a level 1 subdirectory of the cache. +// +// The function works under the assumption that directory entries with one +// character names (except ".") are subdirectories and that there are no other +// subdirectories. +// +// Files ignored: +// - CACHEDIR.TAG +// - stats +// - .nfs* (temporary NFS files that may be left for open but deleted files). +// +// Parameters: +// - dir: The directory to traverse recursively. +// - progress_receiver: Function that will be called for progress updates. +std::vector +get_level_1_files(const std::string& dir, + const ProgressReceiver& progress_receiver); + +} // namespace primary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/CMakeLists.txt ccache-4.5.1/src/storage/secondary/CMakeLists.txt --- ccache-4.2.1/src/storage/secondary/CMakeLists.txt 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,12 @@ +set( + sources + ${CMAKE_CURRENT_SOURCE_DIR}/FileStorage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/HttpStorage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/SecondaryStorage.cpp +) + +if(REDIS_STORAGE_BACKEND) + list(APPEND sources ${CMAKE_CURRENT_SOURCE_DIR}/RedisStorage.cpp) +endif() + +target_sources(ccache_framework PRIVATE ${sources}) diff -Nru ccache-4.2.1/src/storage/secondary/FileStorage.cpp ccache-4.5.1/src/storage/secondary/FileStorage.cpp --- ccache-4.2.1/src/storage/secondary/FileStorage.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/FileStorage.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,194 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "FileStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include // for mode_t + +namespace storage { +namespace secondary { + +namespace { + +class FileStorageBackend : public SecondaryStorage::Backend +{ +public: + FileStorageBackend(const Params& params); + + nonstd::expected, Failure> + get(const Digest& key) override; + + nonstd::expected put(const Digest& key, + const std::string& value, + bool only_if_missing) override; + + nonstd::expected remove(const Digest& key) override; + +private: + enum class Layout { flat, subdirs }; + + const std::string m_dir; + nonstd::optional m_umask; + bool m_update_mtime = false; + Layout m_layout = Layout::subdirs; + + std::string get_entry_path(const Digest& key) const; +}; + +FileStorageBackend::FileStorageBackend(const Params& params) + : m_dir(params.url.path()) +{ + ASSERT(params.url.scheme() == "file"); + if (!params.url.host().empty()) { + throw core::Fatal(FMT( + "invalid file path \"{}\": specifying a host (\"{}\") is not supported", + params.url.str(), + params.url.host())); + } + + for (const auto& attr : params.attributes) { + if (attr.key == "layout") { + if (attr.value == "flat") { + m_layout = Layout::flat; + } else if (attr.value == "subdirs") { + m_layout = Layout::subdirs; + } else { + LOG("Unknown layout: {}", attr.value); + } + } else if (attr.key == "umask") { + m_umask = + util::value_or_throw(util::parse_umask(attr.value)); + } else if (attr.key == "update-mtime") { + m_update_mtime = attr.value == "true"; + } else if (!is_framework_attribute(attr.key)) { + LOG("Unknown attribute: {}", attr.key); + } + } +} + +nonstd::expected, + SecondaryStorage::Backend::Failure> +FileStorageBackend::get(const Digest& key) +{ + const auto path = get_entry_path(key); + const bool exists = Stat::stat(path); + + if (!exists) { + // Don't log failure if the entry doesn't exist. + return nonstd::nullopt; + } + + if (m_update_mtime) { + // Update modification timestamp for potential LRU cleanup by some external + // mechanism. + Util::update_mtime(path); + } + + try { + LOG("Reading {}", path); + return Util::read_file(path); + } catch (const core::Error& e) { + LOG("Failed to read {}: {}", path, e.what()); + return nonstd::make_unexpected(Failure::error); + } +} + +nonstd::expected +FileStorageBackend::put(const Digest& key, + const std::string& value, + const bool only_if_missing) +{ + const auto path = get_entry_path(key); + + if (only_if_missing && Stat::stat(path)) { + LOG("{} already in cache", path); + return false; + } + + { + UmaskScope umask_scope(m_umask); + + const auto dir = Util::dir_name(path); + if (!Util::create_dir(dir)) { + LOG("Failed to create directory {}: {}", dir, strerror(errno)); + return nonstd::make_unexpected(Failure::error); + } + + util::create_cachedir_tag(m_dir); + + LOG("Writing {}", path); + try { + AtomicFile file(path, AtomicFile::Mode::binary); + file.write(value); + file.commit(); + return true; + } catch (const core::Error& e) { + LOG("Failed to write {}: {}", path, e.what()); + return nonstd::make_unexpected(Failure::error); + } + } +} + +nonstd::expected +FileStorageBackend::remove(const Digest& key) +{ + return Util::unlink_safe(get_entry_path(key)); +} + +std::string +FileStorageBackend::get_entry_path(const Digest& key) const +{ + switch (m_layout) { + case Layout::flat: + return FMT("{}/{}", m_dir, key.to_string()); + + case Layout::subdirs: { + const auto key_str = key.to_string(); + const uint8_t digits = 2; + ASSERT(key_str.length() > digits); + return FMT("{}/{:.{}}/{}", m_dir, key_str, digits, &key_str[digits]); + } + } + + ASSERT(false); +} + +} // namespace + +std::unique_ptr +FileStorage::create_backend(const Backend::Params& params) const +{ + return std::make_unique(params); +} + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/FileStorage.hpp ccache-4.5.1/src/storage/secondary/FileStorage.hpp --- ccache-4.2.1/src/storage/secondary/FileStorage.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/FileStorage.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,34 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace storage { +namespace secondary { + +class FileStorage : public SecondaryStorage +{ +public: + std::unique_ptr + create_backend(const Backend::Params& params) const override; +}; + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/HttpStorage.cpp ccache-4.5.1/src/storage/secondary/HttpStorage.cpp --- ccache-4.2.1/src/storage/secondary/HttpStorage.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/HttpStorage.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,287 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "HttpStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace storage { +namespace secondary { + +namespace { + +class HttpStorageBackend : public SecondaryStorage::Backend +{ +public: + HttpStorageBackend(const Params& params); + + nonstd::expected, Failure> + get(const Digest& key) override; + + nonstd::expected put(const Digest& key, + const std::string& value, + bool only_if_missing) override; + + nonstd::expected remove(const Digest& key) override; + +private: + enum class Layout { bazel, flat, subdirs }; + + const std::string m_url_path; + httplib::Client m_http_client; + Layout m_layout = Layout::subdirs; + + std::string get_entry_path(const Digest& key) const; +}; + +std::string +get_url_path(const Url& url) +{ + auto path = url.path(); + if (path.empty() || path.back() != '/') { + path += '/'; + } + return path; +} + +Url +get_partial_url(const Url& from_url) +{ + Url url; + url.scheme(from_url.scheme()); + url.host(from_url.host(), from_url.ip_version()); + if (!from_url.port().empty()) { + url.port(from_url.port()); + } + return url; +} + +std::string +get_url(const Url& url) +{ + if (url.host().empty()) { + throw core::Fatal("A host is required in HTTP storage URL \"{}\"", + url.str()); + } + + // httplib requires a partial URL with just scheme, host and port. + return get_partial_url(url).str(); +} + +HttpStorageBackend::HttpStorageBackend(const Params& params) + : m_url_path(get_url_path(params.url)), + m_http_client(get_url(params.url)) +{ + if (!params.url.user_info().empty()) { + const auto pair = util::split_once(params.url.user_info(), ':'); + if (!pair.second) { + throw core::Fatal("Expected username:password in URL but got \"{}\"", + params.url.user_info()); + } + m_http_client.set_basic_auth(std::string(pair.first).c_str(), + std::string(*pair.second).c_str()); + } + + m_http_client.set_default_headers({ + {"User-Agent", FMT("{}/{}", CCACHE_NAME, CCACHE_VERSION)}, + }); + m_http_client.set_keep_alive(false); + + auto connect_timeout = k_default_connect_timeout; + auto operation_timeout = k_default_operation_timeout; + + for (const auto& attr : params.attributes) { + if (attr.key == "connect-timeout") { + connect_timeout = parse_timeout_attribute(attr.value); + } else if (attr.key == "keep-alive") { + m_http_client.set_keep_alive(attr.value == "true"); + } else if (attr.key == "layout") { + if (attr.value == "bazel") { + m_layout = Layout::bazel; + } else if (attr.value == "flat") { + m_layout = Layout::flat; + } else if (attr.value == "subdirs") { + m_layout = Layout::subdirs; + } else { + LOG("Unknown layout: {}", attr.value); + } + } else if (attr.key == "operation-timeout") { + operation_timeout = parse_timeout_attribute(attr.value); + } else if (!is_framework_attribute(attr.key)) { + LOG("Unknown attribute: {}", attr.key); + } + } + + m_http_client.set_connection_timeout(connect_timeout); + m_http_client.set_read_timeout(operation_timeout); + m_http_client.set_write_timeout(operation_timeout); +} + +nonstd::expected, + SecondaryStorage::Backend::Failure> +HttpStorageBackend::get(const Digest& key) +{ + const auto url_path = get_entry_path(key); + const auto result = m_http_client.Get(url_path.c_str()); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to get {} from http storage: {} ({})", + url_path, + to_string(result.error()), + result.error()); + return nonstd::make_unexpected(Failure::error); + } + + if (result->status < 200 || result->status >= 300) { + // Don't log failure if the entry doesn't exist. + return nonstd::nullopt; + } + + return result->body; +} + +nonstd::expected +HttpStorageBackend::put(const Digest& key, + const std::string& value, + const bool only_if_missing) +{ + const auto url_path = get_entry_path(key); + + if (only_if_missing) { + const auto result = m_http_client.Head(url_path.c_str()); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to check for {} in http storage: {} ({})", + url_path, + to_string(result.error()), + result.error()); + return nonstd::make_unexpected(Failure::error); + } + + if (result->status >= 200 && result->status < 300) { + LOG("Found entry {} already within http storage: status code: {}", + url_path, + result->status); + return false; + } + } + + static const auto content_type = "application/octet-stream"; + const auto result = m_http_client.Put( + url_path.c_str(), value.data(), value.size(), content_type); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to put {} to http storage: {} ({})", + url_path, + to_string(result.error()), + result.error()); + return nonstd::make_unexpected(Failure::error); + } + + if (result->status < 200 || result->status >= 300) { + LOG("Failed to put {} to http storage: status code: {}", + url_path, + result->status); + return nonstd::make_unexpected(Failure::error); + } + + return true; +} + +nonstd::expected +HttpStorageBackend::remove(const Digest& key) +{ + const auto url_path = get_entry_path(key); + const auto result = m_http_client.Delete(url_path.c_str()); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to delete {} from http storage: {} ({})", + url_path, + to_string(result.error()), + result.error()); + return nonstd::make_unexpected(Failure::error); + } + + if (result->status < 200 || result->status >= 300) { + LOG("Failed to delete {} from http storage: status code: {}", + url_path, + result->status); + return nonstd::make_unexpected(Failure::error); + } + + return true; +} + +std::string +HttpStorageBackend::get_entry_path(const Digest& key) const +{ + switch (m_layout) { + case Layout::bazel: { + // Mimic hex representation of a SHA256 hash value. + const auto sha256_hex_size = 64; + static_assert(Digest::size() == 20, "Update below if digest size changes"); + std::string hex_digits = Util::format_base16(key.bytes(), key.size()); + hex_digits.append(hex_digits.data(), sha256_hex_size - hex_digits.size()); + LOG("Translated key {} to Bazel layout ac/{}", key.to_string(), hex_digits); + return FMT("{}ac/{}", m_url_path, hex_digits); + } + + case Layout::flat: + return m_url_path + key.to_string(); + + case Layout::subdirs: { + const auto key_str = key.to_string(); + const uint8_t digits = 2; + ASSERT(key_str.length() > digits); + return FMT("{}/{:.{}}/{}", m_url_path, key_str, digits, &key_str[digits]); + } + } + + ASSERT(false); +} + +} // namespace + +std::unique_ptr +HttpStorage::create_backend(const Backend::Params& params) const +{ + return std::make_unique(params); +} + +void +HttpStorage::redact_secrets(Backend::Params& params) const +{ + auto& url = params.url; + const auto user_info = util::split_once(url.user_info(), ':'); + if (user_info.second) { + url.user_info(FMT("{}:{}", user_info.first, k_redacted_password)); + } +} + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/HttpStorage.hpp ccache-4.5.1/src/storage/secondary/HttpStorage.hpp --- ccache-4.2.1/src/storage/secondary/HttpStorage.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/HttpStorage.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,36 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace storage { +namespace secondary { + +class HttpStorage : public SecondaryStorage +{ +public: + std::unique_ptr + create_backend(const Backend::Params& params) const override; + + void redact_secrets(Backend::Params& params) const override; +}; + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/RedisStorage.cpp ccache-4.5.1/src/storage/secondary/RedisStorage.cpp --- ccache-4.2.1/src/storage/secondary/RedisStorage.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/RedisStorage.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,344 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "RedisStorage.hpp" + +#include +#include +#include +#include +#include +#include + +// Ignore "ISO C++ forbids flexible array member ‘buf’" warning from -Wpedantic. +#ifdef __GNUC__ +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpedantic" +#endif +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable : 4200) +#endif +#include +#ifdef _MSC_VER +# pragma warning(pop) +#endif +#ifdef __GNUC__ +# pragma GCC diagnostic pop +#endif + +#include +#include + +namespace storage { +namespace secondary { + +namespace { + +using RedisContext = std::unique_ptr; +using RedisReply = std::unique_ptr; + +const uint32_t DEFAULT_PORT = 6379; + +class RedisStorageBackend : public SecondaryStorage::Backend +{ +public: + RedisStorageBackend(const SecondaryStorage::Backend::Params& params); + + nonstd::expected, Failure> + get(const Digest& key) override; + + nonstd::expected put(const Digest& key, + const std::string& value, + bool only_if_missing) override; + + nonstd::expected remove(const Digest& key) override; + +private: + const std::string m_prefix; + RedisContext m_context; + + void + connect(const Url& url, uint32_t connect_timeout, uint32_t operation_timeout); + void select_database(const Url& url); + void authenticate(const Url& url); + nonstd::expected redis_command(const char* format, ...); + std::string get_key_string(const Digest& digest) const; +}; + +timeval +to_timeval(const uint32_t ms) +{ + timeval tv; + tv.tv_sec = ms / 1000; + tv.tv_usec = (ms % 1000) * 1000; + return tv; +} + +std::pair, nonstd::optional> +split_user_info(const std::string& user_info) +{ + const auto pair = util::split_once(user_info, ':'); + if (pair.first.empty()) { + // redis://HOST + return {nonstd::nullopt, nonstd::nullopt}; + } else if (pair.second) { + // redis://USERNAME:PASSWORD@HOST + return {std::string(*pair.second), std::string(pair.first)}; + } else { + // redis://PASSWORD@HOST + return {std::string(pair.first), nonstd::nullopt}; + } +} + +RedisStorageBackend::RedisStorageBackend(const Params& params) + : m_prefix("ccache"), // TODO: attribute + m_context(nullptr, redisFree) +{ + const auto& url = params.url; + ASSERT(url.scheme() == "redis"); + + auto connect_timeout = k_default_connect_timeout; + auto operation_timeout = k_default_operation_timeout; + + for (const auto& attr : params.attributes) { + if (attr.key == "connect-timeout") { + connect_timeout = parse_timeout_attribute(attr.value); + } else if (attr.key == "operation-timeout") { + operation_timeout = parse_timeout_attribute(attr.value); + } else if (!is_framework_attribute(attr.key)) { + LOG("Unknown attribute: {}", attr.key); + } + } + + connect(url, connect_timeout.count(), operation_timeout.count()); + select_database(url); + authenticate(url); +} + +inline bool +is_error(int err) +{ + return err != REDIS_OK; +} + +inline bool +is_timeout(int err) +{ +#ifdef REDIS_ERR_TIMEOUT + // Only returned for hiredis version 1.0.0 and above + return err == REDIS_ERR_TIMEOUT; +#else + (void)err; + return false; +#endif +} + +nonstd::expected, + SecondaryStorage::Backend::Failure> +RedisStorageBackend::get(const Digest& key) +{ + const auto key_string = get_key_string(key); + LOG("Redis GET {}", key_string); + const auto reply = redis_command("GET %s", key_string.c_str()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type == REDIS_REPLY_STRING) { + return std::string((*reply)->str, (*reply)->len); + } else if ((*reply)->type == REDIS_REPLY_NIL) { + return nonstd::nullopt; + } else { + LOG("Unknown reply type: {}", (*reply)->type); + return nonstd::make_unexpected(Failure::error); + } +} + +nonstd::expected +RedisStorageBackend::put(const Digest& key, + const std::string& value, + bool only_if_missing) +{ + const auto key_string = get_key_string(key); + + if (only_if_missing) { + LOG("Redis EXISTS {}", key_string); + const auto reply = redis_command("EXISTS %s", key_string.c_str()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type == REDIS_REPLY_INTEGER && (*reply)->integer > 0) { + LOG("Entry {} already in Redis", key_string); + return false; + } else { + LOG("Unknown reply type: {}", (*reply)->type); + } + } + + LOG("Redis SET {} [{} bytes]", key_string, value.size()); + const auto reply = + redis_command("SET %s %b", key_string.c_str(), value.data(), value.size()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type == REDIS_REPLY_STATUS) { + return true; + } else { + LOG("Unknown reply type: {}", (*reply)->type); + return nonstd::make_unexpected(Failure::error); + } +} + +nonstd::expected +RedisStorageBackend::remove(const Digest& key) +{ + const auto key_string = get_key_string(key); + LOG("Redis DEL {}", key_string); + const auto reply = redis_command("DEL %s", key_string.c_str()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type == REDIS_REPLY_INTEGER) { + return (*reply)->integer > 0; + } else { + LOG("Unknown reply type: {}", (*reply)->type); + return nonstd::make_unexpected(Failure::error); + } +} + +void +RedisStorageBackend::connect(const Url& url, + const uint32_t connect_timeout, + const uint32_t operation_timeout) +{ + const std::string host = url.host().empty() ? "localhost" : url.host(); + const uint32_t port = url.port().empty() + ? DEFAULT_PORT + : util::value_or_throw( + util::parse_unsigned(url.port(), 1, 65535, "port")); + ASSERT(url.path().empty() || url.path()[0] == '/'); + + LOG("Redis connecting to {}:{} (connect timeout {} ms)", + url.host(), + port, + connect_timeout); + m_context.reset(redisConnectWithTimeout( + url.host().c_str(), port, to_timeval(connect_timeout))); + + if (!m_context) { + throw Failed("Redis context construction error"); + } + if (is_timeout(m_context->err)) { + throw Failed(FMT("Redis connection timeout: {}", m_context->errstr), + Failure::timeout); + } + if (is_error(m_context->err)) { + throw Failed(FMT("Redis connection error: {}", m_context->errstr)); + } + + LOG("Redis operation timeout set to {} ms", operation_timeout); + if (redisSetTimeout(m_context.get(), to_timeval(operation_timeout)) + != REDIS_OK) { + throw Failed("Failed to set operation timeout"); + } + + LOG_RAW("Redis connection OK"); +} + +void +RedisStorageBackend::select_database(const Url& url) +{ + const uint32_t db_number = + url.path().empty() ? 0 + : util::value_or_throw(util::parse_unsigned( + url.path().substr(1), + 0, + std::numeric_limits::max(), + "db number")); + + if (db_number != 0) { + LOG("Redis SELECT {}", db_number); + const auto reply = + util::value_or_throw(redis_command("SELECT %d", db_number)); + } +} + +void +RedisStorageBackend::authenticate(const Url& url) +{ + const auto password_username_pair = split_user_info(url.user_info()); + const auto& password = password_username_pair.first; + if (password) { + const auto& username = password_username_pair.second; + if (username) { + LOG("Redis AUTH {} {}", *username, k_redacted_password); + util::value_or_throw( + redis_command("AUTH %s %s", username->c_str(), password->c_str())); + } else { + LOG("Redis AUTH {}", k_redacted_password); + util::value_or_throw(redis_command("AUTH %s", password->c_str())); + } + } +} + +nonstd::expected +RedisStorageBackend::redis_command(const char* format, ...) +{ + va_list ap; + va_start(ap, format); + auto reply = + static_cast(redisvCommand(m_context.get(), format, ap)); + va_end(ap); + if (!reply) { + LOG("Redis command failed: {}", m_context->errstr); + return nonstd::make_unexpected(is_timeout(m_context->err) ? Failure::timeout + : Failure::error); + } else if (reply->type == REDIS_REPLY_ERROR) { + LOG("Redis command failed: {}", reply->str); + return nonstd::make_unexpected(Failure::error); + } else { + return RedisReply(reply, freeReplyObject); + } +} + +std::string +RedisStorageBackend::get_key_string(const Digest& digest) const +{ + return FMT("{}:{}", m_prefix, digest.to_string()); +} + +} // namespace + +std::unique_ptr +RedisStorage::create_backend(const Backend::Params& params) const +{ + return std::make_unique(params); +} + +void +RedisStorage::redact_secrets(Backend::Params& params) const +{ + auto& url = params.url; + const auto user_info = util::split_once(url.user_info(), ':'); + if (user_info.second) { + // redis://username:password@host + url.user_info(FMT("{}:{}", user_info.first, k_redacted_password)); + } else if (!user_info.first.empty()) { + // redis://password@host + url.user_info(k_redacted_password); + } +} + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/RedisStorage.hpp ccache-4.5.1/src/storage/secondary/RedisStorage.hpp --- ccache-4.2.1/src/storage/secondary/RedisStorage.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/RedisStorage.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,36 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace storage { +namespace secondary { + +class RedisStorage : public SecondaryStorage +{ +public: + std::unique_ptr + create_backend(const Backend::Params& params) const override; + + void redact_secrets(Backend::Params& params) const override; +}; + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/SecondaryStorage.cpp ccache-4.5.1/src/storage/secondary/SecondaryStorage.cpp --- ccache-4.2.1/src/storage/secondary/SecondaryStorage.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/SecondaryStorage.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,41 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "SecondaryStorage.hpp" + +#include +#include + +namespace storage { +namespace secondary { + +bool +SecondaryStorage::Backend::is_framework_attribute(const std::string& name) +{ + return name == "read-only" || name == "shards" || name == "share-hits"; +} + +std::chrono::milliseconds +SecondaryStorage::Backend::parse_timeout_attribute(const std::string& value) +{ + return std::chrono::milliseconds(util::value_or_throw( + util::parse_unsigned(value, 1, 60 * 1000, "timeout"))); +} + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/secondary/SecondaryStorage.hpp ccache-4.5.1/src/storage/secondary/SecondaryStorage.hpp --- ccache-4.2.1/src/storage/secondary/SecondaryStorage.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/secondary/SecondaryStorage.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,147 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include +#include + +class Digest; + +namespace storage { +namespace secondary { + +constexpr auto k_redacted_password = "********"; +const auto k_default_connect_timeout = std::chrono::milliseconds{100}; +const auto k_default_operation_timeout = std::chrono::milliseconds{10000}; + +// This class defines the API that a secondary storage must implement. +class SecondaryStorage +{ +public: + class Backend + { + public: + struct Attribute + { + std::string key; // Key part. + std::string value; // Value part, percent-decoded. + std::string raw_value; // Value part, not percent-decoded. + }; + + struct Params + { + Url url; + std::vector attributes; + }; + + enum class Failure { + error, // Operation error, e.g. bad parameters or failed connection. + timeout, // Timeout, e.g. due to slow network or server. + }; + + class Failed : public std::runtime_error + { + public: + Failed(Failure failure); + Failed(const std::string& message, Failure failure = Failure::error); + + Failure failure() const; + + private: + Failure m_failure; + }; + + virtual ~Backend() = default; + + // Get the value associated with `key`. Returns the value on success or + // nonstd::nullopt if the entry is not present. + virtual nonstd::expected, Failure> + get(const Digest& key) = 0; + + // Put `value` associated to `key` in the storage. A true `only_if_missing` + // is a hint that the value does not have to be set if already present. + // Returns true if the entry was stored, otherwise false. + virtual nonstd::expected + put(const Digest& key, + const std::string& value, + bool only_if_missing = false) = 0; + + // Remove `key` and its associated value. Returns true if the entry was + // removed, otherwise false. + virtual nonstd::expected remove(const Digest& key) = 0; + + // Determine whether an attribute is handled by the secondary storage + // framework itself. + static bool is_framework_attribute(const std::string& name); + + // Parse a timeout `value`, throwing `Failed` on error. + static std::chrono::milliseconds + parse_timeout_attribute(const std::string& value); + }; + + virtual ~SecondaryStorage() = default; + + // Create an instance of the backend. The instance is created just before the + // first call to a backend method, so the backend constructor can open a + // connection or similar right away if wanted. The method should throw + // `core::Fatal` on fatal configuration error or `Backend::Failed` on + // connection error or timeout. + virtual std::unique_ptr + create_backend(const Backend::Params& parameters) const = 0; + + // Redact secrets in backend parameters, if any. + virtual void redact_secrets(Backend::Params& parameters) const; +}; + +// --- Inline implementations --- + +inline void +SecondaryStorage::redact_secrets( + SecondaryStorage::Backend::Params& /*config*/) const +{ +} + +inline SecondaryStorage::Backend::Failed::Failed(Failure failure) + : Failed("", failure) +{ +} + +inline SecondaryStorage::Backend::Failed::Failed(const std::string& message, + Failure failure) + : std::runtime_error::runtime_error(message), + m_failure(failure) +{ +} + +inline SecondaryStorage::Backend::Failure +SecondaryStorage::Backend::Failed::failure() const +{ + return m_failure; +} + +} // namespace secondary +} // namespace storage diff -Nru ccache-4.2.1/src/storage/Storage.cpp ccache-4.5.1/src/storage/Storage.cpp --- ccache-4.2.1/src/storage/Storage.cpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/Storage.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,570 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Storage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_REDIS_STORAGE_BACKEND +# include +#endif +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace storage { + +const std::unordered_map> + k_secondary_storage_implementations = { + {"file", std::make_shared()}, + {"http", std::make_shared()}, +#ifdef HAVE_REDIS_STORAGE_BACKEND + {"redis", std::make_shared()}, +#endif +}; + +std::string +get_features() +{ + std::vector features; + features.reserve(k_secondary_storage_implementations.size()); + std::transform(k_secondary_storage_implementations.begin(), + k_secondary_storage_implementations.end(), + std::back_inserter(features), + [](auto& entry) { return FMT("{}-storage", entry.first); }); + std::sort(features.begin(), features.end()); + return util::join(features, " "); +} + +struct SecondaryStorageShardConfig +{ + std::string name; + double weight; +}; + +struct SecondaryStorageConfig +{ + std::vector shards; + secondary::SecondaryStorage::Backend::Params params; + bool read_only = false; + bool share_hits = true; +}; + +struct SecondaryStorageBackendEntry +{ + Url url; // With expanded "*". + std::string url_for_logging; // With expanded "*". + std::unique_ptr impl; + bool failed = false; +}; + +struct SecondaryStorageEntry +{ + SecondaryStorageConfig config; + std::string url_for_logging; // With unexpanded "*". + std::shared_ptr storage; + std::vector backends; +}; + +static std::string +to_string(const SecondaryStorageConfig& entry) +{ + std::string result = entry.params.url.str(); + for (const auto& attr : entry.params.attributes) { + result += FMT("|{}={}", attr.key, attr.raw_value); + } + return result; +} + +static SecondaryStorageConfig +parse_storage_config(const nonstd::string_view entry) +{ + const auto parts = + Util::split_into_views(entry, "|", util::Tokenizer::Mode::include_empty); + + if (parts.empty() || parts.front().empty()) { + throw core::Error("secondary storage config must provide a URL: {}", entry); + } + + SecondaryStorageConfig result; + result.params.url = std::string(parts[0]); + // The Url class is parsing the URL object lazily; check if successful. + try { + std::ignore = result.params.url.host(); + } catch (const Url::parse_error& e) { + throw core::Error("Cannot parse URL: {}", e.what()); + } + + if (result.params.url.scheme().empty()) { + throw core::Error("URL scheme must not be empty: {}", entry); + } + + for (size_t i = 1; i < parts.size(); ++i) { + if (parts[i].empty()) { + continue; + } + const auto kv_pair = util::split_once(parts[i], '='); + const auto& key = kv_pair.first; + const auto& raw_value = kv_pair.second.value_or("true"); + const auto value = + util::value_or_throw(util::percent_decode(raw_value)); + if (key == "read-only") { + result.read_only = (value == "true"); + } else if (key == "shards") { + const auto url_str = result.params.url.str(); + if (url_str.find('*') == std::string::npos) { + throw core::Error(R"(Missing "*" in URL when using shards: "{}")", + url_str); + } + for (const auto& shard : util::Tokenizer(value, ",")) { + double weight = 1.0; + nonstd::string_view name; + const auto lp_pos = shard.find('('); + if (lp_pos != nonstd::string_view::npos) { + if (shard.back() != ')') { + throw core::Error("Invalid shard name: \"{}\"", shard); + } + weight = + util::value_or_throw(util::parse_double(std::string( + shard.substr(lp_pos + 1, shard.length() - lp_pos - 2)))); + if (weight < 0.0) { + throw core::Error("Invalid shard weight: \"{}\"", weight); + } + name = shard.substr(0, lp_pos); + } else { + name = shard; + } + + result.shards.push_back({std::string(name), weight}); + } + } else if (key == "share-hits") { + result.share_hits = (value == "true"); + } + + result.params.attributes.push_back( + {std::string(key), value, std::string(raw_value)}); + } + + return result; +} + +static std::vector +parse_storage_configs(const nonstd::string_view& configs) +{ + std::vector result; + for (const auto& config : util::Tokenizer(configs, " ")) { + result.push_back(parse_storage_config(config)); + } + return result; +} + +static std::shared_ptr +get_storage(const Url& url) +{ + const auto it = k_secondary_storage_implementations.find(url.scheme()); + if (it != k_secondary_storage_implementations.end()) { + return it->second; + } else { + return {}; + } +} + +Storage::Storage(const Config& config) : primary(config), m_config(config) +{ +} + +Storage::~Storage() +{ + for (const auto& tmp_file : m_tmp_files) { + Util::unlink_tmp(tmp_file); + } +} + +void +Storage::initialize() +{ + primary.initialize(); + add_secondary_storages(); +} + +void +Storage::finalize() +{ + primary.finalize(); +} + +nonstd::optional +Storage::get(const Digest& key, const core::CacheEntryType type) +{ + MTR_SCOPE("storage", "get"); + + const auto path = primary.get(key, type); + primary.increment_statistic(path ? core::Statistic::primary_storage_hit + : core::Statistic::primary_storage_miss); + if (path) { + if (m_config.reshare()) { + // Temporary optimization until primary storage API has been refactored to + // pass data via memory instead of files. + const bool should_put_in_secondary_storage = + std::any_of(m_secondary_storages.begin(), + m_secondary_storages.end(), + [](const auto& entry) { return !entry->config.read_only; }); + if (should_put_in_secondary_storage) { + std::string value; + try { + value = Util::read_file(*path); + } catch (const core::Error& e) { + LOG("Failed to read {}: {}", *path, e.what()); + return path; // Don't indicate failure since primary storage was OK. + } + put_in_secondary_storage(key, value, true); + } + } + + return path; + } + + const auto value_and_share_hits = get_from_secondary_storage(key); + if (!value_and_share_hits) { + return nonstd::nullopt; + } + const auto& value = value_and_share_hits->first; + const auto& share_hits = value_and_share_hits->second; + + TemporaryFile tmp_file(FMT("{}/tmp.get", m_config.temporary_dir())); + m_tmp_files.push_back(tmp_file.path); + try { + Util::write_file(tmp_file.path, value); + } catch (const core::Error& e) { + throw core::Fatal("Error writing to {}: {}", tmp_file.path, e.what()); + } + + if (share_hits) { + primary.put(key, type, [&](const auto& path) { + try { + Util::copy_file(tmp_file.path, path); + } catch (const core::Error& e) { + LOG("Failed to copy {} to {}: {}", tmp_file.path, path, e.what()); + // Don't indicate failure since get from primary storage was OK. + } + return true; + }); + } + + return tmp_file.path; +} + +bool +Storage::put(const Digest& key, + const core::CacheEntryType type, + const storage::EntryWriter& entry_writer) +{ + MTR_SCOPE("storage", "put"); + + const auto path = primary.put(key, type, entry_writer); + if (!path) { + return false; + } + + // Temporary optimization until primary storage API has been refactored to + // pass data via memory instead of files. + const bool should_put_in_secondary_storage = + std::any_of(m_secondary_storages.begin(), + m_secondary_storages.end(), + [](const auto& entry) { return !entry->config.read_only; }); + if (should_put_in_secondary_storage) { + std::string value; + try { + value = Util::read_file(*path); + } catch (const core::Error& e) { + LOG("Failed to read {}: {}", *path, e.what()); + return true; // Don't indicate failure since primary storage was OK. + } + put_in_secondary_storage(key, value, false); + } + + return true; +} + +void +Storage::remove(const Digest& key, const core::CacheEntryType type) +{ + MTR_SCOPE("storage", "remove"); + + primary.remove(key, type); + remove_from_secondary_storage(key); +} + +bool +Storage::has_secondary_storage() const +{ + return !m_secondary_storages.empty(); +} + +std::string +Storage::get_secondary_storage_config_for_logging() const +{ + auto configs = parse_storage_configs(m_config.secondary_storage()); + for (auto& config : configs) { + const auto storage = get_storage(config.params.url); + if (storage) { + storage->redact_secrets(config.params); + } + } + return util::join(configs, " "); +} + +void +Storage::add_secondary_storages() +{ + const auto configs = parse_storage_configs(m_config.secondary_storage()); + for (const auto& config : configs) { + auto url_for_logging = config.params.url; + url_for_logging.user_info(""); + const auto storage = get_storage(config.params.url); + if (!storage) { + throw core::Error("unknown secondary storage URL: {}", + url_for_logging.str()); + } + m_secondary_storages.push_back(std::make_unique( + SecondaryStorageEntry{config, url_for_logging.str(), storage, {}})); + } +} + +void +Storage::mark_backend_as_failed( + SecondaryStorageBackendEntry& backend_entry, + const secondary::SecondaryStorage::Backend::Failure failure) +{ + // The backend is expected to log details about the error. + backend_entry.failed = true; + primary.increment_statistic( + failure == secondary::SecondaryStorage::Backend::Failure::timeout + ? core::Statistic::secondary_storage_timeout + : core::Statistic::secondary_storage_error); +} + +static double +to_half_open_unit_interval(uint64_t value) +{ + constexpr uint8_t double_significand_bits = 53; + constexpr uint64_t denominator = 1ULL << double_significand_bits; + constexpr uint64_t mask = denominator - 1; + return static_cast(value & mask) / denominator; +} + +static Url +get_shard_url(const Digest& key, + const std::string& url, + const std::vector& shards) +{ + ASSERT(!shards.empty()); + + // This is the "weighted rendezvous hashing" algorithm. + double highest_score = -1.0; + std::string best_shard; + for (const auto& shard_config : shards) { + util::XXH3_64 hash; + hash.update(key.bytes(), key.size()); + hash.update(shard_config.name.data(), shard_config.name.length()); + const double score = to_half_open_unit_interval(hash.digest()); + ASSERT(score >= 0.0 && score < 1.0); + const double weighted_score = + score == 0.0 ? 0.0 : shard_config.weight / -std::log(score); + if (weighted_score > highest_score) { + best_shard = shard_config.name; + highest_score = weighted_score; + } + } + + return util::replace_first(url, "*", best_shard); +} + +SecondaryStorageBackendEntry* +Storage::get_backend(SecondaryStorageEntry& entry, + const Digest& key, + const nonstd::string_view operation_description, + const bool for_writing) +{ + if (for_writing && entry.config.read_only) { + LOG("Not {} {} since it is read-only", + operation_description, + entry.url_for_logging); + return nullptr; + } + + const auto shard_url = + entry.config.shards.empty() + ? entry.config.params.url + : get_shard_url(key, entry.config.params.url.str(), entry.config.shards); + auto backend = + std::find_if(entry.backends.begin(), + entry.backends.end(), + [&](const auto& x) { return x.url.str() == shard_url.str(); }); + + if (backend == entry.backends.end()) { + auto shard_url_for_logging = shard_url; + shard_url_for_logging.user_info(""); + entry.backends.push_back( + {shard_url, shard_url_for_logging.str(), {}, false}); + auto shard_params = entry.config.params; + shard_params.url = shard_url; + try { + entry.backends.back().impl = entry.storage->create_backend(shard_params); + } catch (const secondary::SecondaryStorage::Backend::Failed& e) { + LOG("Failed to construct backend for {}{}", + entry.url_for_logging, + nonstd::string_view(e.what()).empty() ? "" : FMT(": {}", e.what())); + mark_backend_as_failed(entry.backends.back(), e.failure()); + return nullptr; + } + return &entry.backends.back(); + } else if (backend->failed) { + LOG("Not {} {} since it failed earlier", + operation_description, + entry.url_for_logging); + return nullptr; + } else { + return &*backend; + } +} + +nonstd::optional> +Storage::get_from_secondary_storage(const Digest& key) +{ + MTR_SCOPE("secondary_storage", "get"); + + for (const auto& entry : m_secondary_storages) { + auto backend = get_backend(*entry, key, "getting from", false); + if (!backend) { + continue; + } + + Timer timer; + const auto result = backend->impl->get(key); + const auto ms = timer.measure_ms(); + if (!result) { + mark_backend_as_failed(*backend, result.error()); + continue; + } + + const auto& value = *result; + if (value) { + LOG("Retrieved {} from {} ({:.2f} ms)", + key.to_string(), + backend->url_for_logging, + ms); + primary.increment_statistic(core::Statistic::secondary_storage_hit); + return std::make_pair(*value, entry->config.share_hits); + } else { + LOG("No {} in {} ({:.2f} ms)", + key.to_string(), + backend->url_for_logging, + ms); + primary.increment_statistic(core::Statistic::secondary_storage_miss); + } + } + + return nonstd::nullopt; +} + +void +Storage::put_in_secondary_storage(const Digest& key, + const std::string& value, + bool only_if_missing) +{ + MTR_SCOPE("secondary_storage", "put"); + + for (const auto& entry : m_secondary_storages) { + auto backend = get_backend(*entry, key, "putting in", true); + if (!backend) { + continue; + } + + Timer timer; + const auto result = backend->impl->put(key, value, only_if_missing); + const auto ms = timer.measure_ms(); + if (!result) { + // The backend is expected to log details about the error. + mark_backend_as_failed(*backend, result.error()); + continue; + } + + const bool stored = *result; + LOG("{} {} in {} ({:.2f} ms)", + stored ? "Stored" : "Did not have to store", + key.to_string(), + entry->url_for_logging, + ms); + } +} + +void +Storage::remove_from_secondary_storage(const Digest& key) +{ + MTR_SCOPE("secondary_storage", "remove"); + + for (const auto& entry : m_secondary_storages) { + auto backend = get_backend(*entry, key, "removing from", true); + if (!backend) { + continue; + } + + Timer timer; + const auto result = backend->impl->remove(key); + const auto ms = timer.measure_ms(); + if (!result) { + mark_backend_as_failed(*backend, result.error()); + continue; + } + + const bool removed = *result; + if (removed) { + LOG("Removed {} from {} ({:.2f} ms)", + key.to_string(), + entry->url_for_logging, + ms); + } else { + LOG("No {} to remove from {} ({:.2f} ms)", + key.to_string(), + entry->url_for_logging, + ms); + } + } +} + +} // namespace storage diff -Nru ccache-4.2.1/src/storage/Storage.hpp ccache-4.5.1/src/storage/Storage.hpp --- ccache-4.2.1/src/storage/Storage.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/Storage.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,93 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +class Digest; + +namespace storage { + +std::string get_features(); + +struct SecondaryStorageBackendEntry; +struct SecondaryStorageEntry; + +class Storage +{ +public: + Storage(const Config& config); + ~Storage(); + + void initialize(); + void finalize(); + + primary::PrimaryStorage primary; + + // Returns a path to a file containing the value. + nonstd::optional get(const Digest& key, + core::CacheEntryType type); + + bool put(const Digest& key, + core::CacheEntryType type, + const storage::EntryWriter& entry_writer); + + void remove(const Digest& key, core::CacheEntryType type); + + bool has_secondary_storage() const; + std::string get_secondary_storage_config_for_logging() const; + +private: + const Config& m_config; + std::vector> m_secondary_storages; + std::vector m_tmp_files; + + void add_secondary_storages(); + + void + mark_backend_as_failed(SecondaryStorageBackendEntry& backend_entry, + secondary::SecondaryStorage::Backend::Failure failure); + + SecondaryStorageBackendEntry* + get_backend(SecondaryStorageEntry& entry, + const Digest& key, + nonstd::string_view operation_description, + const bool for_writing); + nonstd::optional> + get_from_secondary_storage(const Digest& key); + + void put_in_secondary_storage(const Digest& key, + const std::string& value, + bool only_if_missing); + + void remove_from_secondary_storage(const Digest& key); +}; + +} // namespace storage diff -Nru ccache-4.2.1/src/storage/types.hpp ccache-4.5.1/src/storage/types.hpp --- ccache-4.2.1/src/storage/types.hpp 1970-01-01 00:00:00.000000000 +0000 +++ ccache-4.5.1/src/storage/types.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -0,0 +1,28 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace storage { + +using EntryWriter = std::function; + +} // namespace storage diff -Nru ccache-4.2.1/src/system.hpp ccache-4.5.1/src/system.hpp --- ccache-4.2.1/src/system.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/system.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,212 +0,0 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#ifdef __MINGW32__ -# define __USE_MINGW_ANSI_STDIO 1 -# define __STDC_FORMAT_MACROS 1 -#endif - -#include "config.h" - -#ifdef HAVE_SYS_FILE_H -# include -#endif - -#ifdef HAVE_SYS_MMAN_H -# include -#endif -#include -#include -#ifdef HAVE_SYS_WAIT_H -# include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_DIRENT_H -# include -#endif - -#include - -#ifdef HAVE_STRINGS_H -# include -#endif - -#ifdef HAVE_UNISTD_H -# include -#endif - -#ifdef HAVE_UTIME_H -# include -#elif defined(HAVE_SYS_UTIME_H) -# include -#endif - -#ifdef HAVE_VARARGS_H -# include -#endif - -// AIX/PASE does not properly define usleep within its headers. However, the -// function is available in libc.a. This extern define ensures that it is -// usable within the ccache code base. -#ifdef _AIX -extern "C" int usleep(useconds_t); -#endif - -#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) - -// Buffer size for I/O operations. Should be a multiple of 4 KiB. -const size_t READ_BUFFER_SIZE = 65536; - -#ifndef ESTALE -# define ESTALE -1 -#endif - -#ifdef _WIN32 -# ifndef _WIN32_WINNT -// _WIN32_WINNT is set in the generated header config.h -# error _WIN32_WINNT is undefined -# endif - -# ifdef _MSC_VER -typedef int mode_t; -typedef int pid_t; -# endif - -# ifndef __MINGW32__ -typedef int64_t ssize_t; -# endif - -// Defined in Win32Util.cpp -void usleep(int64_t usec); -struct tm* localtime_r(time_t* _clock, struct tm* _result); - -# ifdef _MSC_VER -int gettimeofday(struct timeval* tp, struct timezone* tzp); -int asprintf(char** strp, const char* fmt, ...); -# endif - -// From: -// http://mesos.apache.org/api/latest/c++/3rdparty_2stout_2include_2stout_2windows_8hpp_source.html -# ifdef _MSC_VER -const mode_t S_IRUSR = mode_t(_S_IREAD); -const mode_t S_IWUSR = mode_t(_S_IWRITE); -# endif - -# ifndef S_IFIFO -# define S_IFIFO 0x1000 -# endif - -# ifndef S_IFBLK -# define S_IFBLK 0x6000 -# endif - -# ifndef S_IFLNK -# define S_IFLNK 0xA000 -# endif - -# ifndef S_ISREG -# define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) -# endif -# ifndef S_ISDIR -# define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) -# endif -# ifndef S_ISFIFO -# define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO) -# endif -# ifndef S_ISCHR -# define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR) -# endif -# ifndef S_ISLNK -# define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK) -# endif -# ifndef S_ISBLK -# define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK) -# endif - -# include -# include -# include -# define NOMINMAX 1 -# include -# define mkdir(a, b) _mkdir(a) -# define execv(a, b) \ - do_not_call_execv_on_windows // to protect against incidental use of MinGW - // execv -# define strncasecmp _strnicmp -# define strcasecmp _stricmp - -# ifdef _MSC_VER -# define PATH_MAX MAX_PATH -# endif - -# ifdef _MSC_VER -# define DLLIMPORT __declspec(dllimport) -# else -# define DLLIMPORT -# endif - -# define STDIN_FILENO 0 -# define STDOUT_FILENO 1 -# define STDERR_FILENO 2 -# define DIR_DELIM_CH '\\' -# define PATH_DELIM ";" -#else -# define DLLIMPORT -# define DIR_DELIM_CH '/' -# define PATH_DELIM ":" -#endif - -DLLIMPORT extern char** environ; - -// Work with silly DOS binary open. -#ifndef O_BINARY -# define O_BINARY 0 -#endif - -#if defined(HAVE_SYS_MMAN_H) && defined(HAVE_PTHREAD_MUTEXATTR_SETPSHARED) -# define INODE_CACHE_SUPPORTED -#endif - -// Workaround for missing std::is_trivially_copyable in GCC < 5. -#if __GNUG__ && __GNUC__ < 5 -# define IS_TRIVIALLY_COPYABLE(T) __has_trivial_copy(T) -#else -# define IS_TRIVIALLY_COPYABLE(T) std::is_trivially_copyable::value -#endif - -// GCC version of a couple of standard C++ attributes -#ifdef __GNUC__ -# define nodiscard gnu::warn_unused_result -# define maybe_unused gnu::unused -#endif diff -Nru ccache-4.2.1/src/TemporaryFile.cpp ccache-4.5.1/src/TemporaryFile.cpp --- ccache-4.2.1/src/TemporaryFile.cpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/TemporaryFile.cpp 2021-11-17 19:31:58.000000000 +0000 @@ -20,6 +20,8 @@ #include "Util.hpp" +#include + #ifdef _WIN32 # include "third_party/win32/mktemp.h" #endif @@ -61,7 +63,7 @@ fd = Fd(mkstemp(&path[0])); #endif if (!fd) { - throw Fatal( + throw core::Fatal( "Failed to create temporary file for {}: {}", path, strerror(errno)); } diff -Nru ccache-4.2.1/src/TemporaryFile.hpp ccache-4.5.1/src/TemporaryFile.hpp --- ccache-4.2.1/src/TemporaryFile.hpp 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/TemporaryFile.hpp 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // diff -Nru ccache-4.2.1/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm ccache-4.5.1/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm --- ccache-4.2.1/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm 2021-11-17 19:31:58.000000000 +0000 @@ -2421,8 +2421,8 @@ movzx r8d, r8b shl rax, 32 add r8, rax - vmovd xmm3, r9 - vmovd xmm4, r8 + vmovq xmm3, r9 + vmovq xmm4, r8 vpunpcklqdq xmm3, xmm3, xmm4 vmovaps xmm2, xmmword ptr [BLAKE3_IV] vmovups xmm8, xmmword ptr [rdx] @@ -2516,8 +2516,8 @@ mov r10, qword ptr [rsp+78H] shl rax, 32 add r8, rax - vmovd xmm3, r9 - vmovd xmm4, r8 + vmovq xmm3, r9 + vmovq xmm4, r8 vpunpcklqdq xmm3, xmm3, xmm4 vmovaps xmm2, xmmword ptr [BLAKE3_IV] vmovups xmm8, xmmword ptr [rdx] diff -Nru ccache-4.2.1/src/third_party/blake3/blake3.c ccache-4.5.1/src/third_party/blake3/blake3.c --- ccache-4.2.1/src/third_party/blake3/blake3.c 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3.c 2021-11-17 19:31:58.000000000 +0000 @@ -5,9 +5,7 @@ #include "blake3.h" #include "blake3_impl.h" -const char * blake3_version(void) { - return BLAKE3_VERSION_STRING; -} +const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], uint8_t flags) { @@ -342,12 +340,18 @@ uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, chunk_counter, flags, cv_array); + assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, // compress_subtree_wide() returns more than 2 chaining values. Condense // them into 2 by forming parent nodes repeatedly. uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - while (num_cvs > 2) { + // The second half of this loop condition is always true, and we just + // asserted it above. But GCC can't tell that it's always true, and if NDEBUG + // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious + // warnings here. GCC 8.5 is particularly sensitive, so if you're changing + // this code, test it against that version. + while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { num_cvs = compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); diff -Nru ccache-4.2.1/src/third_party/blake3/blake3_dispatch.c ccache-4.5.1/src/third_party/blake3/blake3_dispatch.c --- ccache-4.2.1/src/third_party/blake3/blake3_dispatch.c 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3_dispatch.c 2021-11-17 19:31:58.000000000 +0000 @@ -232,7 +232,7 @@ #endif #endif -#if defined(BLAKE3_USE_NEON) +#if BLAKE3_USE_NEON == 1 blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out); return; @@ -269,7 +269,7 @@ } #endif #endif -#if defined(BLAKE3_USE_NEON) +#if BLAKE3_USE_NEON == 1 return 4; #endif return 1; diff -Nru ccache-4.2.1/src/third_party/blake3/blake3.h ccache-4.5.1/src/third_party/blake3/blake3.h --- ccache-4.2.1/src/third_party/blake3/blake3.h 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3.h 2021-11-17 19:31:58.000000000 +0000 @@ -8,13 +8,12 @@ extern "C" { #endif -#define BLAKE3_VERSION_STRING "0.3.7" +#define BLAKE3_VERSION_STRING "1.2.0" #define BLAKE3_KEY_LEN 32 #define BLAKE3_OUT_LEN 32 #define BLAKE3_BLOCK_LEN 64 #define BLAKE3_CHUNK_LEN 1024 #define BLAKE3_MAX_DEPTH 54 -#define BLAKE3_MAX_SIMD_DEGREE 16 // This struct is a private implementation detail. It has to be here because // it's part of blake3_hasher below. @@ -39,12 +38,12 @@ uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; } blake3_hasher; -const char * blake3_version(void); +const char *blake3_version(void); void blake3_hasher_init(blake3_hasher *self); void blake3_hasher_init_keyed(blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]); void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); -void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, +void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, size_t context_len); void blake3_hasher_update(blake3_hasher *self, const void *input, size_t input_len); diff -Nru ccache-4.2.1/src/third_party/blake3/blake3_impl.h ccache-4.5.1/src/third_party/blake3/blake3_impl.h --- ccache-4.2.1/src/third_party/blake3/blake3_impl.h 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3_impl.h 2021-11-17 19:31:58.000000000 +0000 @@ -38,6 +38,10 @@ #define IS_X86_32 #endif +#if defined(__aarch64__) || defined(_M_ARM64) +#define IS_AARCH64 +#endif + #if defined(IS_X86) #if defined(_MSC_VER) #include @@ -45,9 +49,18 @@ #include #endif +#if !defined(BLAKE3_USE_NEON) + // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness + #if defined(IS_AARCH64) + #define BLAKE3_USE_NEON 1 + #else + #define BLAKE3_USE_NEON 0 + #endif +#endif + #if defined(IS_X86) #define MAX_SIMD_DEGREE 16 -#elif defined(BLAKE3_USE_NEON) +#elif BLAKE3_USE_NEON == 1 #define MAX_SIMD_DEGREE 4 #else #define MAX_SIMD_DEGREE 1 @@ -257,7 +270,7 @@ #endif #endif -#if defined(BLAKE3_USE_NEON) +#if BLAKE3_USE_NEON == 1 void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, diff -Nru ccache-4.2.1/src/third_party/blake3/blake3_neon.c ccache-4.5.1/src/third_party/blake3/blake3_neon.c --- ccache-4.2.1/src/third_party/blake3/blake3_neon.c 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3_neon.c 2021-11-17 19:31:58.000000000 +0000 @@ -2,7 +2,12 @@ #include -// TODO: This is probably incorrect for big-endian ARM. How should that work? +#ifdef __ARM_BIG_ENDIAN +#error "This implementation only supports little-endian ARM." +// It might be that all we need for big-endian support here is to get the loads +// and stores right, but step zero would be finding a way to test it in CI. +#endif + INLINE uint32x4_t loadu_128(const uint8_t src[16]) { // vld1q_u32 has alignment requirements. Don't use it. uint32x4_t x; diff -Nru ccache-4.2.1/src/third_party/blake3/blake3_sse2_x86-64_unix.S ccache-4.5.1/src/third_party/blake3/blake3_sse2_x86-64_unix.S --- ccache-4.2.1/src/third_party/blake3/blake3_sse2_x86-64_unix.S 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3_sse2_x86-64_unix.S 2021-11-17 19:31:58.000000000 +0000 @@ -1704,7 +1704,7 @@ pshufd xmm15, xmm11, 0x93 shl rax, 0x20 or rax, 0x40 - movd xmm3, rax + movq xmm3, rax movdqa xmmword ptr [rsp+0x20], xmm3 movaps xmm3, xmmword ptr [rsp] movaps xmm11, xmmword ptr [rsp+0x10] @@ -1917,7 +1917,7 @@ movaps xmm2, xmmword ptr [BLAKE3_IV+rip] shl rax, 32 or rax, 64 - movd xmm12, rax + movq xmm12, rax movdqa xmm3, xmm13 punpcklqdq xmm3, xmm12 movups xmm4, xmmword ptr [r8+rdx-0x40] diff -Nru ccache-4.2.1/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S ccache-4.5.1/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S --- ccache-4.2.1/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S 2021-11-17 19:31:58.000000000 +0000 @@ -1715,7 +1715,7 @@ pshufd xmm15, xmm11, 0x93 shl rax, 0x20 or rax, 0x40 - movd xmm3, rax + movq xmm3, rax movdqa xmmword ptr [rsp+0x20], xmm3 movaps xmm3, xmmword ptr [rsp] movaps xmm11, xmmword ptr [rsp+0x10] @@ -1928,7 +1928,7 @@ movaps xmm2, xmmword ptr [BLAKE3_IV+rip] shl rax, 32 or rax, 64 - movd xmm12, rax + movq xmm12, rax movdqa xmm3, xmm13 punpcklqdq xmm3, xmm12 movups xmm4, xmmword ptr [r8+rdx-0x40] @@ -2137,10 +2137,10 @@ por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm14 pshufd xmm8, xmm8, 0x78 punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 @@ -2268,10 +2268,10 @@ por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm14 pshufd xmm8, xmm8, 0x78 punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 diff -Nru ccache-4.2.1/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm ccache-4.5.1/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm --- ccache-4.2.1/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm 2021-11-17 19:31:58.000000000 +0000 @@ -2139,10 +2139,10 @@ por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK] + por xmm8, xmm14 pshufd xmm8, xmm8, 78H punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 @@ -2271,10 +2271,10 @@ por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK] + por xmm8, xmm14 pshufd xmm8, xmm8, 78H punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 diff -Nru ccache-4.2.1/src/third_party/blake3/CMakeLists.txt ccache-4.5.1/src/third_party/blake3/CMakeLists.txt --- ccache-4.2.1/src/third_party/blake3/CMakeLists.txt 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/blake3/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -17,17 +17,17 @@ set(compile_flags "${others_flags}") endif() + if(MSVC) + set(suffix "_x86-64_windows_msvc.asm") + elseif(WIN32) + set(suffix "_x86-64_windows_gnu.S") + else() + set(suffix "_x86-64_unix.S") + endif() + # First check if it's possible to use the assembler variant for the feature. string(TOUPPER "have_asm_${feature}" have_feature) if(NOT DEFINED "${have_feature}" AND CMAKE_SIZEOF_VOID_P EQUAL 8) - if(MSVC) - set(suffix "_x86-64_windows_msvc.asm") - elseif(WIN32) - set(suffix "_x86-64_windows_gnu.S") - else() - set(suffix "_x86-64_unix.S") - endif() - if(NOT CMAKE_REQUIRED_QUIET) message(STATUS "Performing Test ${have_feature}") endif() diff -Nru ccache-4.2.1/src/third_party/CMakeLists.txt ccache-4.5.1/src/third_party/CMakeLists.txt --- ccache-4.2.1/src/third_party/CMakeLists.txt 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/CMakeLists.txt 2021-11-17 19:31:58.000000000 +0000 @@ -1,17 +1,17 @@ -add_library(third_party_lib STATIC base32hex.c format.cpp xxhash.c) +add_library(third_party STATIC base32hex.c format.cpp httplib.cpp url.cpp xxhash.c) if(NOT MSVC) - target_sources(third_party_lib PRIVATE getopt_long.c) + target_sources(third_party PRIVATE getopt_long.c) else() - target_sources(third_party_lib PRIVATE win32/getopt.c) - target_compile_definitions(third_party_lib PUBLIC -DSTATIC_GETOPT) + target_sources(third_party PRIVATE win32/getopt.c) + target_compile_definitions(third_party PUBLIC -DSTATIC_GETOPT) endif() if(WIN32) - target_sources(third_party_lib PRIVATE win32/mktemp.c) + target_sources(third_party PRIVATE win32/mktemp.c) endif () if(ENABLE_TRACING) - target_sources(third_party_lib PRIVATE minitrace.c) + target_sources(third_party PRIVATE minitrace.c) endif() set(xxhdispatchtest [=[ @@ -31,47 +31,28 @@ CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CMAKE_CURRENT_SOURCE_DIR}" COMPILE_DEFINITIONS "-DXXH_STATIC_LINKING_ONLY") -target_compile_definitions(third_party_lib INTERFACE "-DXXH_STATIC_LINKING_ONLY") +target_compile_definitions(third_party INTERFACE "-DXXH_STATIC_LINKING_ONLY") if(USE_XXH_DISPATCH) - target_sources(third_party_lib PRIVATE xxh_x86dispatch.c) - target_compile_definitions(third_party_lib INTERFACE "-DUSE_XXH_DISPATCH") + target_sources(third_party PRIVATE xxh_x86dispatch.c) + target_compile_definitions(third_party INTERFACE "-DUSE_XXH_DISPATCH") endif() # Treat third party headers as system files (no warning for those headers). target_include_directories( - third_party_lib - PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} SYSTEM) + third_party + PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} SYSTEM +) -target_link_libraries(third_party_lib PRIVATE standard_settings) -target_link_libraries(third_party_lib INTERFACE blake3) +target_link_libraries(third_party PRIVATE standard_settings) +target_link_libraries(third_party INTERFACE blake3) -# These warnings are enabled by default even without e.g. -Wall, but we don't -# want them in third_party. -if(CMAKE_CXX_COMPILER_ID MATCHES "^GNU|Clang$") - target_compile_options( - third_party_lib - PRIVATE - $<$:-Wno-implicit-function-declaration - -Wno-int-conversion>) -endif() - -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - target_compile_options( - third_party_lib - PRIVATE $<$:-Wno-attributes>) +if(WIN32) + target_link_libraries(third_party PRIVATE ws2_32) endif() # Silence warning from winbase.h due to /Zc:preprocessor. if(MSVC) - target_compile_options( - third_party_lib - PRIVATE /wd5105) -endif() - -# The headers are included from the rest of the project, so turn off warnings as -# required. -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(third_party_lib INTERFACE -Wno-shadow) + target_compile_options(third_party PRIVATE /wd5105) endif() add_subdirectory(blake3) diff -Nru ccache-4.2.1/src/third_party/fmt/core.h ccache-4.5.1/src/third_party/fmt/core.h --- ccache-4.2.1/src/third_party/fmt/core.h 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/fmt/core.h 2021-11-17 19:31:58.000000000 +0000 @@ -1,4 +1,4 @@ -// Formatting library for C++ - the core API +// Formatting library for C++ - the core API for char/UTF-8 // // Copyright (c) 2012 - present, Victor Zverovich // All rights reserved. @@ -10,15 +10,13 @@ #include // std::FILE #include -#include #include -#include +#include #include #include -#include // The fmt library version in the form major * 10000 + minor * 100 + patch. -#define FMT_VERSION 70103 +#define FMT_VERSION 80001 #ifdef __clang__ # define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) @@ -26,16 +24,12 @@ # define FMT_CLANG_VERSION 0 #endif -#if defined(__GNUC__) && !defined(__clang__) +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) # define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# define FMT_GCC_PRAGMA(arg) _Pragma(arg) #else # define FMT_GCC_VERSION 0 -#endif - -#if defined(__INTEL_COMPILER) -# define FMT_ICC_VERSION __INTEL_COMPILER -#else -# define FMT_ICC_VERSION 0 +# define FMT_GCC_PRAGMA(arg) #endif #if __cplusplus >= 201103L || defined(__GXX_EXPERIMENTAL_CXX0X__) @@ -44,6 +38,12 @@ # define FMT_HAS_GXX_CXX11 0 #endif +#if defined(__INTEL_COMPILER) +# define FMT_ICC_VERSION __INTEL_COMPILER +#else +# define FMT_ICC_VERSION 0 +#endif + #ifdef __NVCC__ # define FMT_NVCC __NVCC__ #else @@ -52,10 +52,10 @@ #ifdef _MSC_VER # define FMT_MSC_VER _MSC_VER -# define FMT_SUPPRESS_MSC_WARNING(n) __pragma(warning(suppress : n)) +# define FMT_MSC_WARNING(...) __pragma(warning(__VA_ARGS__)) #else # define FMT_MSC_VER 0 -# define FMT_SUPPRESS_MSC_WARNING(n) +# define FMT_MSC_WARNING(...) #endif #ifdef __has_feature @@ -64,7 +64,8 @@ # define FMT_HAS_FEATURE(x) 0 #endif -#if defined(__has_include) && !defined(__INTELLISENSE__) && \ +#if defined(__has_include) && \ + (!defined(__INTELLISENSE__) || FMT_MSC_VER > 1900) && \ (!FMT_ICC_VERSION || FMT_ICC_VERSION >= 1600) # define FMT_HAS_INCLUDE(x) __has_include(x) #else @@ -95,10 +96,26 @@ # define FMT_CONSTEXPR constexpr # define FMT_CONSTEXPR_DECL constexpr #else -# define FMT_CONSTEXPR inline +# define FMT_CONSTEXPR # define FMT_CONSTEXPR_DECL #endif +// Check if constexpr std::char_traits<>::compare,length is supported. +#if defined(__GLIBCXX__) +# if __cplusplus >= 201703L && defined(_GLIBCXX_RELEASE) && \ + _GLIBCXX_RELEASE >= 7 // GCC 7+ libstdc++ has _GLIBCXX_RELEASE. +# define FMT_CONSTEXPR_CHAR_TRAITS constexpr +# endif +#elif defined(_LIBCPP_VERSION) && __cplusplus >= 201703L && \ + _LIBCPP_VERSION >= 4000 +# define FMT_CONSTEXPR_CHAR_TRAITS constexpr +#elif FMT_MSC_VER >= 1914 && _MSVC_LANG >= 201703L +# define FMT_CONSTEXPR_CHAR_TRAITS constexpr +#endif +#ifndef FMT_CONSTEXPR_CHAR_TRAITS +# define FMT_CONSTEXPR_CHAR_TRAITS +#endif + #ifndef FMT_OVERRIDE # if FMT_HAS_FEATURE(cxx_override_control) || \ (FMT_GCC_VERSION >= 408 && FMT_HAS_GXX_CXX11) || FMT_MSC_VER >= 1900 @@ -149,25 +166,40 @@ # define FMT_NORETURN #endif -#ifndef FMT_DEPRECATED -# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900 -# define FMT_DEPRECATED [[deprecated]] +#ifndef FMT_MAYBE_UNUSED +# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused) +# define FMT_MAYBE_UNUSED [[maybe_unused]] # else -# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__) -# define FMT_DEPRECATED __attribute__((deprecated)) -# elif FMT_MSC_VER -# define FMT_DEPRECATED __declspec(deprecated) -# else -# define FMT_DEPRECATED /* deprecated */ -# endif +# define FMT_MAYBE_UNUSED # endif #endif -// Workaround broken [[deprecated]] in the Intel, PGI and NVCC compilers. -#if FMT_ICC_VERSION || defined(__PGI) || FMT_NVCC -# define FMT_DEPRECATED_ALIAS +#if __cplusplus == 201103L || __cplusplus == 201402L +# if defined(__INTEL_COMPILER) || defined(__PGI) +# define FMT_FALLTHROUGH +# elif defined(__clang__) +# define FMT_FALLTHROUGH [[clang::fallthrough]] +# elif FMT_GCC_VERSION >= 700 && \ + (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) +# define FMT_FALLTHROUGH [[gnu::fallthrough]] +# else +# define FMT_FALLTHROUGH +# endif +#elif FMT_HAS_CPP17_ATTRIBUTE(fallthrough) || \ + (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +# define FMT_FALLTHROUGH [[fallthrough]] #else -# define FMT_DEPRECATED_ALIAS FMT_DEPRECATED +# define FMT_FALLTHROUGH +#endif + +#ifndef FMT_USE_FLOAT +# define FMT_USE_FLOAT 1 +#endif +#ifndef FMT_USE_DOUBLE +# define FMT_USE_DOUBLE 1 +#endif +#ifndef FMT_USE_LONG_DOUBLE +# define FMT_USE_LONG_DOUBLE 1 #endif #ifndef FMT_INLINE @@ -180,7 +212,7 @@ #ifndef FMT_USE_INLINE_NAMESPACES # if FMT_HAS_FEATURE(cxx_inline_namespaces) || FMT_GCC_VERSION >= 404 || \ - (FMT_MSC_VER >= 1900 && !_MANAGED) + (FMT_MSC_VER >= 1900 && (!defined(_MANAGED) || !_MANAGED)) # define FMT_USE_INLINE_NAMESPACES 1 # else # define FMT_USE_INLINE_NAMESPACES 0 @@ -197,41 +229,45 @@ # define FMT_INLINE_NAMESPACE namespace # define FMT_END_NAMESPACE \ } \ - using namespace v7; \ + using namespace v8; \ } # endif # define FMT_BEGIN_NAMESPACE \ namespace fmt { \ - FMT_INLINE_NAMESPACE v7 { + FMT_INLINE_NAMESPACE v8 { +#endif + +#ifndef FMT_MODULE_EXPORT +# define FMT_MODULE_EXPORT +# define FMT_MODULE_EXPORT_BEGIN +# define FMT_MODULE_EXPORT_END +# define FMT_BEGIN_DETAIL_NAMESPACE namespace detail { +# define FMT_END_DETAIL_NAMESPACE } #endif #if !defined(FMT_HEADER_ONLY) && defined(_WIN32) -# define FMT_CLASS_API FMT_SUPPRESS_MSC_WARNING(4275) +# define FMT_CLASS_API FMT_MSC_WARNING(suppress : 4275) # ifdef FMT_EXPORT # define FMT_API __declspec(dllexport) -# define FMT_EXTERN_TEMPLATE_API FMT_API -# define FMT_EXPORTED # elif defined(FMT_SHARED) # define FMT_API __declspec(dllimport) -# define FMT_EXTERN_TEMPLATE_API FMT_API # endif #else # define FMT_CLASS_API +# if defined(FMT_EXPORT) || defined(FMT_SHARED) +# if defined(__GNUC__) || defined(__clang__) +# define FMT_API __attribute__((visibility("default"))) +# endif +# endif #endif #ifndef FMT_API # define FMT_API #endif -#ifndef FMT_EXTERN_TEMPLATE_API -# define FMT_EXTERN_TEMPLATE_API -#endif -#ifndef FMT_INSTANTIATION_DEF_API -# define FMT_INSTANTIATION_DEF_API FMT_API -#endif -#ifndef FMT_HEADER_ONLY -# define FMT_EXTERN extern +#if FMT_GCC_VERSION +# define FMT_GCC_VISIBILITY_HIDDEN __attribute__((visibility("hidden"))) #else -# define FMT_EXTERN +# define FMT_GCC_VISIBILITY_HIDDEN #endif // libc++ supports string_view in pre-c++17. @@ -248,38 +284,81 @@ #ifndef FMT_UNICODE # define FMT_UNICODE !FMT_MSC_VER #endif -#if FMT_UNICODE && FMT_MSC_VER -# pragma execution_character_set("utf-8") + +#ifndef FMT_CONSTEVAL +# if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) && \ + __cplusplus > 201703L) || \ + (defined(__cpp_consteval) && \ + !FMT_MSC_VER) // consteval is broken in MSVC. +# define FMT_CONSTEVAL consteval +# define FMT_HAS_CONSTEVAL +# else +# define FMT_CONSTEVAL +# endif +#endif + +#ifndef FMT_USE_NONTYPE_TEMPLATE_PARAMETERS +# if defined(__cpp_nontype_template_args) && \ + ((FMT_GCC_VERSION >= 903 && __cplusplus >= 201709L) || \ + __cpp_nontype_template_args >= 201911L) +# define FMT_USE_NONTYPE_TEMPLATE_PARAMETERS 1 +# else +# define FMT_USE_NONTYPE_TEMPLATE_PARAMETERS 0 +# endif +#endif + +// Enable minimal optimizations for more compact code in debug mode. +FMT_GCC_PRAGMA("GCC push_options") +#ifndef __OPTIMIZE__ +FMT_GCC_PRAGMA("GCC optimize(\"Og\")") #endif FMT_BEGIN_NAMESPACE +FMT_MODULE_EXPORT_BEGIN // Implementations of enable_if_t and other metafunctions for older systems. -template +template using enable_if_t = typename std::enable_if::type; -template +template using conditional_t = typename std::conditional::type; template using bool_constant = std::integral_constant; template using remove_reference_t = typename std::remove_reference::type; template -using remove_const_t = typename std::remove_const::type; -template using remove_cvref_t = typename std::remove_cv>::type; template struct type_identity { using type = T; }; template using type_identity_t = typename type_identity::type; -struct monostate {}; +struct monostate { + constexpr monostate() {} +}; + +// Suppress "unused variable" warnings with the method described in +// https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/. +// (void)var does not work on many Intel compilers. +template FMT_CONSTEXPR void ignore_unused(const T&...) {} // An enable_if helper to be used in template parameters which results in much // shorter symbols: https://godbolt.org/z/sWw4vP. Extra parentheses are needed // to workaround a bug in MSVC 2019 (see #1140 and #1186). -#define FMT_ENABLE_IF(...) enable_if_t<(__VA_ARGS__), int> = 0 +#ifdef FMT_DOC +# define FMT_ENABLE_IF(...) +#else +# define FMT_ENABLE_IF(...) enable_if_t<(__VA_ARGS__), int> = 0 +#endif + +FMT_BEGIN_DETAIL_NAMESPACE -namespace detail { +constexpr FMT_INLINE auto is_constant_evaluated() FMT_NOEXCEPT -> bool { +#ifdef __cpp_lib_is_constant_evaluated + return std::is_constant_evaluated(); +#else + return false; +#endif +} -// A helper function to suppress "conditional expression is constant" warnings. -template constexpr T const_check(T value) { return value; } +// A function to suppress "conditional expression is constant" warnings. +template constexpr auto const_check(T value) -> T { return value; } FMT_NORETURN FMT_API void assert_fail(const char* file, int line, const char* message); @@ -287,7 +366,8 @@ #ifndef FMT_ASSERT # ifdef NDEBUG // FMT_ASSERT is not empty to avoid -Werror=empty-body. -# define FMT_ASSERT(condition, message) ((void)0) +# define FMT_ASSERT(condition, message) \ + ::fmt::ignore_unused((condition), (message)) # else # define FMT_ASSERT(condition, message) \ ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \ @@ -312,38 +392,39 @@ # define FMT_USE_INT128 1 using int128_t = __int128_t; using uint128_t = __uint128_t; +template inline auto convert_for_visit(T value) -> T { + return value; +} #else # define FMT_USE_INT128 0 #endif #if !FMT_USE_INT128 -struct int128_t {}; -struct uint128_t {}; +enum class int128_t {}; +enum class uint128_t {}; +// Reduce template instantiations. +template inline auto convert_for_visit(T) -> monostate { + return {}; +} #endif // Casts a nonnegative integer to unsigned. template -FMT_CONSTEXPR typename std::make_unsigned::type to_unsigned(Int value) { +FMT_CONSTEXPR auto to_unsigned(Int value) -> + typename std::make_unsigned::type { FMT_ASSERT(value >= 0, "negative value"); return static_cast::type>(value); } -FMT_SUPPRESS_MSC_WARNING(4566) constexpr unsigned char micro[] = "\u00B5"; +FMT_MSC_WARNING(suppress : 4566) constexpr unsigned char micro[] = "\u00B5"; -template constexpr bool is_unicode() { - return FMT_UNICODE || sizeof(Char) != 1 || - (sizeof(micro) == 3 && micro[0] == 0xC2 && micro[1] == 0xB5); +constexpr auto is_utf8() -> bool { + // Avoid buggy sign extensions in MSVC's constant evaluation mode. + // https://developercommunity.visualstudio.com/t/C-difference-in-behavior-for-unsigned/1233612 + using uchar = unsigned char; + return FMT_UNICODE || (sizeof(micro) == 3 && uchar(micro[0]) == 0xC2 && + uchar(micro[1]) == 0xB5); } - -#ifdef __cpp_char8_t -using char8_type = char8_t; -#else -enum char8_type : unsigned char {}; -#endif -} // namespace detail - -#ifdef FMT_USE_INTERNAL -namespace internal = detail; // DEPRECATED -#endif +FMT_END_DETAIL_NAMESPACE /** An implementation of ``std::basic_string_view`` for pre-C++17. It provides a @@ -374,11 +455,15 @@ the size with ``std::char_traits::length``. \endrst */ -#if __cplusplus >= 201703L // C++17's char_traits::length() is constexpr. - FMT_CONSTEXPR -#endif - basic_string_view(const Char* s) - : data_(s), size_(std::char_traits::length(s)) {} + FMT_CONSTEXPR_CHAR_TRAITS + FMT_INLINE + basic_string_view(const Char* s) : data_(s) { + if (detail::const_check(std::is_same::value && + !detail::is_constant_evaluated())) + size_ = std::strlen(reinterpret_cast(s)); + else + size_ = std::char_traits::length(s); + } /** Constructs a string reference from a ``std::basic_string`` object. */ template @@ -393,15 +478,17 @@ size_(s.size()) {} /** Returns a pointer to the string data. */ - constexpr const Char* data() const { return data_; } + constexpr auto data() const -> const Char* { return data_; } /** Returns the string size. */ - constexpr size_t size() const { return size_; } + constexpr auto size() const -> size_t { return size_; } - constexpr iterator begin() const { return data_; } - constexpr iterator end() const { return data_ + size_; } + constexpr auto begin() const -> iterator { return data_; } + constexpr auto end() const -> iterator { return data_ + size_; } - constexpr const Char& operator[](size_t pos) const { return data_[pos]; } + constexpr auto operator[](size_t pos) const -> const Char& { + return data_[pos]; + } FMT_CONSTEXPR void remove_prefix(size_t n) { data_ += n; @@ -409,7 +496,7 @@ } // Lexicographically compare this string reference to other. - int compare(basic_string_view other) const { + FMT_CONSTEXPR_CHAR_TRAITS auto compare(basic_string_view other) const -> int { size_t str_size = size_ < other.size_ ? size_ : other.size_; int result = std::char_traits::compare(data_, other.data_, str_size); if (result == 0) @@ -417,72 +504,53 @@ return result; } - friend bool operator==(basic_string_view lhs, basic_string_view rhs) { + FMT_CONSTEXPR_CHAR_TRAITS friend auto operator==(basic_string_view lhs, + basic_string_view rhs) + -> bool { return lhs.compare(rhs) == 0; } - friend bool operator!=(basic_string_view lhs, basic_string_view rhs) { + friend auto operator!=(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) != 0; } - friend bool operator<(basic_string_view lhs, basic_string_view rhs) { + friend auto operator<(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) < 0; } - friend bool operator<=(basic_string_view lhs, basic_string_view rhs) { + friend auto operator<=(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) <= 0; } - friend bool operator>(basic_string_view lhs, basic_string_view rhs) { + friend auto operator>(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) > 0; } - friend bool operator>=(basic_string_view lhs, basic_string_view rhs) { + friend auto operator>=(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) >= 0; } }; using string_view = basic_string_view; -using wstring_view = basic_string_view; /** Specifies if ``T`` is a character type. Can be specialized by users. */ template struct is_char : std::false_type {}; template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; - -/** - \rst - Returns a string view of `s`. In order to add custom string type support to - {fmt} provide an overload of `to_string_view` for it in the same namespace as - the type for the argument-dependent lookup to work. - - **Example**:: - namespace my_ns { - inline string_view to_string_view(const my_string& s) { - return {s.data(), s.length()}; - } - } - std::string message = fmt::format(my_string("The answer is {}"), 42); - \endrst - */ +// Returns a string view of `s`. template ::value)> -inline basic_string_view to_string_view(const Char* s) { +FMT_INLINE auto to_string_view(const Char* s) -> basic_string_view { return s; } - template -inline basic_string_view to_string_view( - const std::basic_string& s) { +inline auto to_string_view(const std::basic_string& s) + -> basic_string_view { return s; } - template -inline basic_string_view to_string_view(basic_string_view s) { +constexpr auto to_string_view(basic_string_view s) + -> basic_string_view { return s; } - template >::value)> -inline basic_string_view to_string_view(detail::std_string_view s) { +inline auto to_string_view(detail::std_string_view s) + -> basic_string_view { return s; } @@ -494,13 +562,15 @@ struct is_compile_string : std::is_base_of {}; template ::value)> -constexpr basic_string_view to_string_view(const S& s) { - return s; +constexpr auto to_string_view(const S& s) + -> basic_string_view { + return basic_string_view(s); } -namespace detail { +FMT_BEGIN_DETAIL_NAMESPACE + void to_string_view(...); -using fmt::v7::to_string_view; +using fmt::v8::to_string_view; // Specifies whether S is a string type convertible to fmt::basic_string_view. // It should be a constexpr function but MSVC 2017 fails to compile it in @@ -534,7 +604,7 @@ // This function is intentionally not constexpr to give a compile-time error. FMT_NORETURN FMT_API void on_error(const char* message); }; -} // namespace detail +FMT_END_DETAIL_NAMESPACE /** String's character type. */ template using char_t = typename detail::char_t_impl::type; @@ -543,16 +613,7 @@ \rst Parsing context consisting of a format string range being parsed and an argument counter for automatic indexing. - - You can use one of the following type aliases for common character types: - - +-----------------------+-------------------------------------+ - | Type | Definition | - +=======================+=====================================+ - | format_parse_context | basic_format_parse_context | - +-----------------------+-------------------------------------+ - | wformat_parse_context | basic_format_parse_context | - +-----------------------+-------------------------------------+ + You can use the ``format_parse_context`` type alias for ``char`` instead. \endrst */ template @@ -574,12 +635,16 @@ Returns an iterator to the beginning of the format string range being parsed. */ - constexpr iterator begin() const FMT_NOEXCEPT { return format_str_.begin(); } + constexpr auto begin() const FMT_NOEXCEPT -> iterator { + return format_str_.begin(); + } /** Returns an iterator past the end of the format string range being parsed. */ - constexpr iterator end() const FMT_NOEXCEPT { return format_str_.end(); } + constexpr auto end() const FMT_NOEXCEPT -> iterator { + return format_str_.end(); + } /** Advances the begin iterator to ``it``. */ FMT_CONSTEXPR void advance_to(iterator it) { @@ -590,7 +655,7 @@ Reports an error if using the manual argument indexing; otherwise returns the next argument index and switches to the automatic indexing. */ - FMT_CONSTEXPR int next_arg_id() { + FMT_CONSTEXPR auto next_arg_id() -> int { // Don't check if the argument id is valid to avoid overhead and because it // will be checked during formatting anyway. if (next_arg_id_ >= 0) return next_arg_id_++; @@ -615,11 +680,10 @@ ErrorHandler::on_error(message); } - constexpr ErrorHandler error_handler() const { return *this; } + constexpr auto error_handler() const -> ErrorHandler { return *this; } }; using format_parse_context = basic_format_parse_context; -using wformat_parse_context = basic_format_parse_context; template class basic_format_arg; template class basic_format_args; @@ -643,11 +707,14 @@ template struct is_contiguous> : std::true_type {}; -namespace detail { +class appender; + +FMT_BEGIN_DETAIL_NAMESPACE // Extracts a reference to the container from back_insert_iterator. template -inline Container& get_container(std::back_insert_iterator it) { +inline auto get_container(std::back_insert_iterator it) + -> Container& { using bi_iterator = std::back_insert_iterator; struct accessor : bi_iterator { accessor(bi_iterator iter) : bi_iterator(iter) {} @@ -656,6 +723,23 @@ return *accessor(it).container; } +template +FMT_CONSTEXPR auto copy_str(InputIt begin, InputIt end, OutputIt out) + -> OutputIt { + while (begin != end) *out++ = static_cast(*begin++); + return out; +} + +template ::value)> +FMT_CONSTEXPR auto copy_str(const Char* begin, const Char* end, Char* out) + -> Char* { + if (is_constant_evaluated()) + return copy_str(begin, end, out); + auto size = to_unsigned(end - begin); + memcpy(out, begin, size); + return out + size; +} + /** \rst A contiguous memory buffer with an optional growing ability. It is an internal @@ -670,7 +754,7 @@ protected: // Don't initialize ptr_ since it is not accessed to save a few cycles. - FMT_SUPPRESS_MSC_WARNING(26495) + FMT_MSC_WARNING(suppress : 26495) buffer(size_t sz) FMT_NOEXCEPT : size_(sz), capacity_(sz) {} buffer(T* p = nullptr, size_t sz = 0, size_t cap = 0) FMT_NOEXCEPT @@ -679,6 +763,7 @@ capacity_(cap) {} ~buffer() = default; + buffer(buffer&&) = default; /** Sets the buffer data and capacity. */ void set(T* buf_data, size_t buf_capacity) FMT_NOEXCEPT { @@ -696,23 +781,23 @@ buffer(const buffer&) = delete; void operator=(const buffer&) = delete; - T* begin() FMT_NOEXCEPT { return ptr_; } - T* end() FMT_NOEXCEPT { return ptr_ + size_; } + auto begin() FMT_NOEXCEPT -> T* { return ptr_; } + auto end() FMT_NOEXCEPT -> T* { return ptr_ + size_; } - const T* begin() const FMT_NOEXCEPT { return ptr_; } - const T* end() const FMT_NOEXCEPT { return ptr_ + size_; } + auto begin() const FMT_NOEXCEPT -> const T* { return ptr_; } + auto end() const FMT_NOEXCEPT -> const T* { return ptr_ + size_; } /** Returns the size of this buffer. */ - size_t size() const FMT_NOEXCEPT { return size_; } + auto size() const FMT_NOEXCEPT -> size_t { return size_; } /** Returns the capacity of this buffer. */ - size_t capacity() const FMT_NOEXCEPT { return capacity_; } + auto capacity() const FMT_NOEXCEPT -> size_t { return capacity_; } /** Returns a pointer to the buffer data. */ - T* data() FMT_NOEXCEPT { return ptr_; } + auto data() FMT_NOEXCEPT -> T* { return ptr_; } /** Returns a pointer to the buffer data. */ - const T* data() const FMT_NOEXCEPT { return ptr_; } + auto data() const FMT_NOEXCEPT -> const T* { return ptr_; } /** Clears this buffer. */ void clear() { size_ = 0; } @@ -740,16 +825,16 @@ /** Appends data to the end of the buffer. */ template void append(const U* begin, const U* end); - template T& operator[](I index) { return ptr_[index]; } - template const T& operator[](I index) const { + template auto operator[](I index) -> T& { return ptr_[index]; } + template auto operator[](I index) const -> const T& { return ptr_[index]; } }; struct buffer_traits { explicit buffer_traits(size_t) {} - size_t count() const { return 0; } - size_t limit(size_t size) { return size; } + auto count() const -> size_t { return 0; } + auto limit(size_t size) -> size_t { return size; } }; class fixed_buffer_traits { @@ -759,8 +844,8 @@ public: explicit fixed_buffer_traits(size_t limit) : limit_(limit) {} - size_t count() const { return count_; } - size_t limit(size_t size) { + auto count() const -> size_t { return count_; } + auto limit(size_t size) -> size_t { size_t n = limit_ > count_ ? limit_ - count_ : 0; count_ += size; return size < n ? size : n; @@ -779,20 +864,25 @@ void grow(size_t) final FMT_OVERRIDE { if (this->size() == buffer_size) flush(); } - void flush(); + + void flush() { + auto size = this->size(); + this->clear(); + out_ = copy_str(data_, data_ + this->limit(size), out_); + } public: explicit iterator_buffer(OutputIt out, size_t n = buffer_size) - : Traits(n), - buffer(data_, 0, buffer_size), - out_(out) {} + : Traits(n), buffer(data_, 0, buffer_size), out_(out) {} + iterator_buffer(iterator_buffer&& other) + : Traits(other), buffer(data_, 0, buffer_size), out_(other.out_) {} ~iterator_buffer() { flush(); } - OutputIt out() { + auto out() -> OutputIt { flush(); return out_; } - size_t count() const { return Traits::count() + this->size(); } + auto count() const -> size_t { return Traits::count() + this->size(); } }; template class iterator_buffer final : public buffer { @@ -802,7 +892,7 @@ public: explicit iterator_buffer(T* out, size_t = 0) : buffer(out, 0, ~size_t()) {} - T* out() { return &*this->end(); } + auto out() -> T* { return &*this->end(); } }; // A buffer that writes to a container with the contiguous storage. @@ -825,7 +915,7 @@ : buffer(c.size()), container_(c) {} explicit iterator_buffer(std::back_insert_iterator out, size_t = 0) : iterator_buffer(get_container(out)) {} - std::back_insert_iterator out() { + auto out() -> std::back_insert_iterator { return std::back_inserter(container_); } }; @@ -847,48 +937,24 @@ public: counting_buffer() : buffer(data_, 0, buffer_size) {} - size_t count() { return count_ + this->size(); } + auto count() -> size_t { return count_ + this->size(); } }; -// An output iterator that appends to the buffer. -// It is used to reduce symbol sizes for the common case. template -class buffer_appender : public std::back_insert_iterator> { - using base = std::back_insert_iterator>; - - public: - explicit buffer_appender(buffer& buf) : base(buf) {} - buffer_appender(base it) : base(it) {} - - buffer_appender& operator++() { - base::operator++(); - return *this; - } - - buffer_appender operator++(int) { - buffer_appender tmp = *this; - ++*this; - return tmp; - } -}; +using buffer_appender = conditional_t::value, appender, + std::back_insert_iterator>>; -// Maps an output iterator into a buffer. +// Maps an output iterator to a buffer. template -iterator_buffer get_buffer(OutputIt); -template buffer& get_buffer(buffer_appender); - -template OutputIt get_buffer_init(OutputIt out) { - return out; -} -template buffer& get_buffer_init(buffer_appender out) { - return get_container(out); +auto get_buffer(OutputIt out) -> iterator_buffer { + return iterator_buffer(out); } template auto get_iterator(Buffer& buf) -> decltype(buf.out()) { return buf.out(); } -template buffer_appender get_iterator(buffer& buf) { +template auto get_iterator(buffer& buf) -> buffer_appender { return buffer_appender(buf); } @@ -898,9 +964,9 @@ }; // Specifies if T has an enabled fallback_formatter specialization. -template +template using has_fallback_formatter = - std::is_constructible>; + std::is_constructible>; struct view {}; @@ -925,8 +991,8 @@ template arg_data(const U&... init) : args_{T(named_args_, NUM_NAMED_ARGS), init...} {} arg_data(const arg_data& other) = delete; - const T* args() const { return args_ + 1; } - named_arg_info* named_args() { return named_args_; } + auto args() const -> const T* { return args_ + 1; } + auto named_args() -> named_arg_info* { return named_args_; } }; template @@ -935,42 +1001,47 @@ T args_[NUM_ARGS != 0 ? NUM_ARGS : +1]; template - FMT_INLINE arg_data(const U&... init) : args_{init...} {} - FMT_INLINE const T* args() const { return args_; } - FMT_INLINE std::nullptr_t named_args() { return nullptr; } + FMT_CONSTEXPR FMT_INLINE arg_data(const U&... init) : args_{init...} {} + FMT_CONSTEXPR FMT_INLINE auto args() const -> const T* { return args_; } + FMT_CONSTEXPR FMT_INLINE auto named_args() -> std::nullptr_t { + return nullptr; + } }; template inline void init_named_args(named_arg_info*, int, int) {} -template +template struct is_named_arg : std::false_type {}; +template struct is_statically_named_arg : std::false_type {}; + +template +struct is_named_arg> : std::true_type {}; + +template ::value)> void init_named_args(named_arg_info* named_args, int arg_count, int named_arg_count, const T&, const Tail&... args) { init_named_args(named_args, arg_count + 1, named_arg_count, args...); } -template +template ::value)> void init_named_args(named_arg_info* named_args, int arg_count, - int named_arg_count, const named_arg& arg, - const Tail&... args) { + int named_arg_count, const T& arg, const Tail&... args) { named_args[named_arg_count++] = {arg.name, arg_count}; init_named_args(named_args, arg_count + 1, named_arg_count, args...); } template -FMT_INLINE void init_named_args(std::nullptr_t, int, int, const Args&...) {} - -template struct is_named_arg : std::false_type {}; - -template -struct is_named_arg> : std::true_type {}; +FMT_CONSTEXPR FMT_INLINE void init_named_args(std::nullptr_t, int, int, + const Args&...) {} -template constexpr size_t count() { return B ? 1 : 0; } -template constexpr size_t count() { +template constexpr auto count() -> size_t { return B ? 1 : 0; } +template constexpr auto count() -> size_t { return (B1 ? 1 : 0) + count(); } -template constexpr size_t count_named_args() { +template constexpr auto count_named_args() -> size_t { return count::value...>(); } @@ -1051,6 +1122,7 @@ using char_type = typename Context::char_type; union { + monostate no_value; int int_value; unsigned uint_value; long long long_long_value; @@ -1068,19 +1140,23 @@ named_arg_value named_args; }; - constexpr FMT_INLINE value(int val = 0) : int_value(val) {} + constexpr FMT_INLINE value() : no_value() {} + constexpr FMT_INLINE value(int val) : int_value(val) {} constexpr FMT_INLINE value(unsigned val) : uint_value(val) {} - FMT_INLINE value(long long val) : long_long_value(val) {} - FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {} + constexpr FMT_INLINE value(long long val) : long_long_value(val) {} + constexpr FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {} FMT_INLINE value(int128_t val) : int128_value(val) {} FMT_INLINE value(uint128_t val) : uint128_value(val) {} FMT_INLINE value(float val) : float_value(val) {} FMT_INLINE value(double val) : double_value(val) {} FMT_INLINE value(long double val) : long_double_value(val) {} - FMT_INLINE value(bool val) : bool_value(val) {} - FMT_INLINE value(char_type val) : char_value(val) {} - FMT_INLINE value(const char_type* val) { string.data = val; } - FMT_INLINE value(basic_string_view val) { + constexpr FMT_INLINE value(bool val) : bool_value(val) {} + constexpr FMT_INLINE value(char_type val) : char_value(val) {} + FMT_CONSTEXPR FMT_INLINE value(const char_type* val) { + string.data = val; + if (is_constant_evaluated()) string.size = {}; + } + FMT_CONSTEXPR FMT_INLINE value(basic_string_view val) { string.data = val.data(); string.size = val.size(); } @@ -1088,7 +1164,7 @@ FMT_INLINE value(const named_arg_info* args, size_t size) : named_args{args, size} {} - template FMT_INLINE value(const T& val) { + template FMT_CONSTEXPR FMT_INLINE value(const T& val) { custom.value = &val; // Get the formatter type through the context to allow different contexts // have different extension points, e.g. `formatter` for `format` and @@ -1112,7 +1188,7 @@ }; template -FMT_CONSTEXPR basic_format_arg make_arg(const T& value); +FMT_CONSTEXPR auto make_arg(const T& value) -> basic_format_arg; // To minimize the number of types we need to deal with, long is translated // either to int or to long long depending on its size. @@ -1126,36 +1202,52 @@ template struct arg_mapper { using char_type = typename Context::char_type; - FMT_CONSTEXPR int map(signed char val) { return val; } - FMT_CONSTEXPR unsigned map(unsigned char val) { return val; } - FMT_CONSTEXPR int map(short val) { return val; } - FMT_CONSTEXPR unsigned map(unsigned short val) { return val; } - FMT_CONSTEXPR int map(int val) { return val; } - FMT_CONSTEXPR unsigned map(unsigned val) { return val; } - FMT_CONSTEXPR long_type map(long val) { return val; } - FMT_CONSTEXPR ulong_type map(unsigned long val) { return val; } - FMT_CONSTEXPR long long map(long long val) { return val; } - FMT_CONSTEXPR unsigned long long map(unsigned long long val) { return val; } - FMT_CONSTEXPR int128_t map(int128_t val) { return val; } - FMT_CONSTEXPR uint128_t map(uint128_t val) { return val; } - FMT_CONSTEXPR bool map(bool val) { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(signed char val) -> int { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned char val) -> unsigned { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(short val) -> int { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned short val) -> unsigned { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(int val) -> int { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned val) -> unsigned { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(long val) -> long_type { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned long val) -> ulong_type { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(long long val) -> long long { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned long long val) + -> unsigned long long { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(int128_t val) -> int128_t { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(uint128_t val) -> uint128_t { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(bool val) -> bool { return val; } template ::value)> - FMT_CONSTEXPR char_type map(T val) { + FMT_CONSTEXPR FMT_INLINE auto map(T val) -> char_type { static_assert( std::is_same::value || std::is_same::value, "mixing character types is disallowed"); return val; } - FMT_CONSTEXPR float map(float val) { return val; } - FMT_CONSTEXPR double map(double val) { return val; } - FMT_CONSTEXPR long double map(long double val) { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(float val) -> float { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(double val) -> double { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(long double val) -> long double { + return val; + } - FMT_CONSTEXPR const char_type* map(char_type* val) { return val; } - FMT_CONSTEXPR const char_type* map(const char_type* val) { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(char_type* val) -> const char_type* { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(const char_type* val) -> const char_type* { + return val; + } template ::value)> - FMT_CONSTEXPR basic_string_view map(const T& val) { + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) + -> basic_string_view { static_assert(std::is_same>::value, "mixing character types is disallowed"); return to_string_view(val); @@ -1164,8 +1256,9 @@ FMT_ENABLE_IF( std::is_constructible, T>::value && !is_string::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR basic_string_view map(const T& val) { + !has_fallback_formatter::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) + -> basic_string_view { return basic_string_view(val); } template < @@ -1174,31 +1267,40 @@ std::is_constructible, T>::value && !std::is_constructible, T>::value && !is_string::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR basic_string_view map(const T& val) { + !has_fallback_formatter::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) + -> basic_string_view { return std_string_view(val); } - FMT_CONSTEXPR const char* map(const signed char* val) { + FMT_CONSTEXPR FMT_INLINE auto map(const signed char* val) -> const char* { static_assert(std::is_same::value, "invalid string type"); return reinterpret_cast(val); } - FMT_CONSTEXPR const char* map(const unsigned char* val) { + FMT_CONSTEXPR FMT_INLINE auto map(const unsigned char* val) -> const char* { static_assert(std::is_same::value, "invalid string type"); return reinterpret_cast(val); } - FMT_CONSTEXPR const char* map(signed char* val) { + FMT_CONSTEXPR FMT_INLINE auto map(signed char* val) -> const char* { const auto* const_val = val; return map(const_val); } - FMT_CONSTEXPR const char* map(unsigned char* val) { + FMT_CONSTEXPR FMT_INLINE auto map(unsigned char* val) -> const char* { const auto* const_val = val; return map(const_val); } - FMT_CONSTEXPR const void* map(void* val) { return val; } - FMT_CONSTEXPR const void* map(const void* val) { return val; } - FMT_CONSTEXPR const void* map(std::nullptr_t val) { return val; } - template FMT_CONSTEXPR int map(const T*) { + FMT_CONSTEXPR FMT_INLINE auto map(void* val) -> const void* { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(const void* val) -> const void* { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(std::nullptr_t val) -> const void* { + return val; + } + + // We use SFINAE instead of a const T* parameter to avoid conflicting with + // the C array overload. + template + FMT_CONSTEXPR auto map(T) -> enable_if_t::value, int> { // Formatting of arbitrary pointers is disallowed. If you want to output // a pointer cast it to "void *" or "const void *". In particular, this // forbids formatting of "[const] volatile char *" which is printed as bool @@ -1207,11 +1309,16 @@ return 0; } + template + FMT_CONSTEXPR FMT_INLINE auto map(const T (&values)[N]) -> const T (&)[N] { + return values; + } + template ::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR auto map(const T& val) + !has_fallback_formatter::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) -> decltype(std::declval().map( static_cast::type>(val))) { return map(static_cast::type>(val)); @@ -1219,18 +1326,18 @@ template ::value && !is_char::value && (has_formatter::value || - has_fallback_formatter::value))> - FMT_CONSTEXPR const T& map(const T& val) { + has_fallback_formatter::value))> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) -> const T& { return val; } - template - FMT_CONSTEXPR auto map(const named_arg& val) - -> decltype(std::declval().map(val.value)) { - return map(val.value); + template ::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& named_arg) + -> decltype(std::declval().map(named_arg.value)) { + return map(named_arg.value); } - unformattable map(...) { return {}; } + auto map(...) -> unformattable { return {}; } }; // A type constant after applying arg_mapper. @@ -1244,7 +1351,35 @@ enum { max_packed_args = 62 / packed_arg_bits }; enum : unsigned long long { is_unpacked_bit = 1ULL << 63 }; enum : unsigned long long { has_named_args_bit = 1ULL << 62 }; -} // namespace detail + +FMT_END_DETAIL_NAMESPACE + +// An output iterator that appends to a buffer. +// It is used to reduce symbol sizes for the common case. +class appender : public std::back_insert_iterator> { + using base = std::back_insert_iterator>; + + template + friend auto get_buffer(appender out) -> detail::buffer& { + return detail::get_container(out); + } + + public: + using std::back_insert_iterator>::back_insert_iterator; + appender(base it) : base(it) {} + using _Unchecked_type = appender; // Mark iterator as checked. + + auto operator++() -> appender& { + base::operator++(); + return *this; + } + + auto operator++(int) -> appender { + auto tmp = *this; + ++*this; + return tmp; + } +}; // A formatting argument. It is a trivially copyable/constructible type to // allow storage in basic_memory_buffer. @@ -1254,8 +1389,8 @@ detail::type type_; template - friend FMT_CONSTEXPR basic_format_arg detail::make_arg( - const T& value); + friend FMT_CONSTEXPR auto detail::make_arg(const T& value) + -> basic_format_arg; template friend FMT_CONSTEXPR auto visit_format_arg(Visitor&& vis, @@ -1293,10 +1428,12 @@ return type_ != detail::type::none_type; } - detail::type type() const { return type_; } + auto type() const -> detail::type { return type_; } - bool is_integral() const { return detail::is_integral_type(type_); } - bool is_arithmetic() const { return detail::is_arithmetic_type(type_); } + auto is_integral() const -> bool { return detail::is_integral_type(type_); } + auto is_arithmetic() const -> bool { + return detail::is_arithmetic_type(type_); + } }; /** @@ -1307,9 +1444,8 @@ \endrst */ template -FMT_CONSTEXPR_DECL FMT_INLINE auto visit_format_arg( +FMT_CONSTEXPR FMT_INLINE auto visit_format_arg( Visitor&& vis, const basic_format_arg& arg) -> decltype(vis(0)) { - using char_type = typename Context::char_type; switch (arg.type_) { case detail::type::none_type: break; @@ -1321,16 +1457,10 @@ return vis(arg.value_.long_long_value); case detail::type::ulong_long_type: return vis(arg.value_.ulong_long_value); -#if FMT_USE_INT128 case detail::type::int128_type: - return vis(arg.value_.int128_value); + return vis(detail::convert_for_visit(arg.value_.int128_value)); case detail::type::uint128_type: - return vis(arg.value_.uint128_value); -#else - case detail::type::int128_type: - case detail::type::uint128_type: - break; -#endif + return vis(detail::convert_for_visit(arg.value_.uint128_value)); case detail::type::bool_type: return vis(arg.value_.bool_value); case detail::type::char_type: @@ -1344,8 +1474,8 @@ case detail::type::cstring_type: return vis(arg.value_.string.data); case detail::type::string_type: - return vis(basic_string_view(arg.value_.string.data, - arg.value_.string.size)); + using sv = basic_string_view; + return vis(sv(arg.value_.string.data, arg.value_.string.size)); case detail::type::pointer_type: return vis(arg.value_.pointer); case detail::type::custom_type: @@ -1354,14 +1484,22 @@ return vis(monostate()); } -template struct formattable : std::false_type {}; +FMT_BEGIN_DETAIL_NAMESPACE -namespace detail { +template +auto copy_str(InputIt begin, InputIt end, appender out) -> appender { + get_container(out).append(begin, end); + return out; +} +#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500 // A workaround for gcc 4.8 to make void_t work in a SFINAE context. template struct void_t_impl { using type = void; }; template using void_t = typename detail::void_t_impl::type; +#else +template using void_t = void; +#endif template struct is_output_iterator : std::false_type {}; @@ -1384,9 +1522,8 @@ template struct is_contiguous_back_insert_iterator> : is_contiguous {}; -template -struct is_contiguous_back_insert_iterator> - : std::true_type {}; +template <> +struct is_contiguous_back_insert_iterator : std::true_type {}; // A type-erased reference to an std::locale to avoid heavy include. class locale_ref { @@ -1394,97 +1531,52 @@ const void* locale_; // A type-erased pointer to std::locale. public: - locale_ref() : locale_(nullptr) {} + constexpr locale_ref() : locale_(nullptr) {} template explicit locale_ref(const Locale& loc); explicit operator bool() const FMT_NOEXCEPT { return locale_ != nullptr; } - template Locale get() const; + template auto get() const -> Locale; }; -template constexpr unsigned long long encode_types() { return 0; } +template constexpr auto encode_types() -> unsigned long long { + return 0; +} template -constexpr unsigned long long encode_types() { +constexpr auto encode_types() -> unsigned long long { return static_cast(mapped_type_constant::value) | (encode_types() << packed_arg_bits); } template -FMT_CONSTEXPR basic_format_arg make_arg(const T& value) { +FMT_CONSTEXPR auto make_arg(const T& value) -> basic_format_arg { basic_format_arg arg; arg.type_ = mapped_type_constant::value; arg.value_ = arg_mapper().map(value); return arg; } -template int check(unformattable) { - static_assert( - formattable(), - "Cannot format an argument. To make type T formattable provide a " - "formatter specialization: https://fmt.dev/latest/api.html#udt"); - return 0; -} -template inline const U& check(const U& val) { - return val; -} - // The type template parameter is there to avoid an ODR violation when using // a fallback formatter in one translation unit and an implicit conversion in // another (not recommended). template -inline value make_arg(const T& val) { - return check(arg_mapper().map(val)); +FMT_CONSTEXPR FMT_INLINE auto make_arg(const T& val) -> value { + const auto& arg = arg_mapper().map(val); + static_assert( + !std::is_same::value, + "Cannot format an argument. To make type T formattable provide a " + "formatter specialization: https://fmt.dev/latest/api.html#udt"); + return {arg}; } template -inline basic_format_arg make_arg(const T& value) { +inline auto make_arg(const T& value) -> basic_format_arg { return make_arg(value); } - -template struct is_reference_wrapper : std::false_type {}; -template -struct is_reference_wrapper> : std::true_type {}; - -template const T& unwrap(const T& v) { return v; } -template const T& unwrap(const std::reference_wrapper& v) { - return static_cast(v); -} - -class dynamic_arg_list { - // Workaround for clang's -Wweak-vtables. Unlike for regular classes, for - // templates it doesn't complain about inability to deduce single translation - // unit for placing vtable. So storage_node_base is made a fake template. - template struct node { - virtual ~node() = default; - std::unique_ptr> next; - }; - - template struct typed_node : node<> { - T value; - - template - FMT_CONSTEXPR typed_node(const Arg& arg) : value(arg) {} - - template - FMT_CONSTEXPR typed_node(const basic_string_view& arg) - : value(arg.data(), arg.size()) {} - }; - - std::unique_ptr> head_; - - public: - template const T& push(const Arg& arg) { - auto new_node = std::unique_ptr>(new typed_node(arg)); - auto& value = new_node->value; - new_node->next = std::move(head_); - head_ = std::move(new_node); - return value; - } -}; -} // namespace detail +FMT_END_DETAIL_NAMESPACE // Formatting context. template class basic_format_context { @@ -1503,46 +1595,59 @@ using parse_context_type = basic_format_parse_context; template using formatter_type = formatter; + basic_format_context(basic_format_context&&) = default; basic_format_context(const basic_format_context&) = delete; void operator=(const basic_format_context&) = delete; /** Constructs a ``basic_format_context`` object. References to the arguments are stored in the object so make sure they have appropriate lifetimes. */ - basic_format_context(OutputIt out, - basic_format_args ctx_args, - detail::locale_ref loc = detail::locale_ref()) + constexpr basic_format_context( + OutputIt out, basic_format_args ctx_args, + detail::locale_ref loc = detail::locale_ref()) : out_(out), args_(ctx_args), loc_(loc) {} - format_arg arg(int id) const { return args_.get(id); } - format_arg arg(basic_string_view name) { return args_.get(name); } - int arg_id(basic_string_view name) { return args_.get_id(name); } - const basic_format_args& args() const { return args_; } + constexpr auto arg(int id) const -> format_arg { return args_.get(id); } + FMT_CONSTEXPR auto arg(basic_string_view name) -> format_arg { + return args_.get(name); + } + FMT_CONSTEXPR auto arg_id(basic_string_view name) -> int { + return args_.get_id(name); + } + auto args() const -> const basic_format_args& { + return args_; + } - detail::error_handler error_handler() { return {}; } + FMT_CONSTEXPR auto error_handler() -> detail::error_handler { return {}; } void on_error(const char* message) { error_handler().on_error(message); } // Returns an iterator to the beginning of the output range. - iterator out() { return out_; } + FMT_CONSTEXPR auto out() -> iterator { return out_; } // Advances the begin iterator to ``it``. void advance_to(iterator it) { if (!detail::is_back_insert_iterator()) out_ = it; } - detail::locale_ref locale() { return loc_; } + FMT_CONSTEXPR auto locale() -> detail::locale_ref { return loc_; } }; template using buffer_context = basic_format_context, Char>; using format_context = buffer_context; -using wformat_context = buffer_context; // Workaround an alias issue: https://stackoverflow.com/q/62767544/471164. #define FMT_BUFFER_CONTEXT(Char) \ basic_format_context, Char> +template +using is_formattable = bool_constant< + !std::is_same>().map( + std::declval())), + detail::unformattable>::value && + !detail::has_fallback_formatter::value>; + /** \rst An array of references to arguments. It can be implicitly converted into @@ -1579,7 +1684,7 @@ : 0); public: - format_arg_store(const Args&... args) + FMT_CONSTEXPR FMT_INLINE format_arg_store(const Args&... args) : #if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 basic_format_args(*this), @@ -1600,36 +1705,16 @@ \endrst */ template -inline format_arg_store make_format_args( - const Args&... args) { - return {args...}; -} - -/** - \rst - Constructs a `~fmt::format_arg_store` object that contains references - to arguments and can be implicitly converted to `~fmt::format_args`. - If ``format_str`` is a compile-time string then `make_args_checked` checks - its validity at compile time. - \endrst - */ -template > -inline auto make_args_checked(const S& format_str, - const remove_reference_t&... args) - -> format_arg_store, remove_reference_t...> { - static_assert( - detail::count<( - std::is_base_of>::value && - std::is_reference::value)...>() == 0, - "passing views as lvalues is disallowed"); - detail::check_format_string(format_str); +constexpr auto make_format_args(const Args&... args) + -> format_arg_store { return {args...}; } /** \rst - Returns a named argument to be used in a formatting function. It should only - be used in a call to a formatting function. + Returns a named argument to be used in a formatting function. + It should only be used in a call to a formatting function or + `dynamic_format_arg_store::push_back`. **Example**:: @@ -1637,234 +1722,63 @@ \endrst */ template -inline detail::named_arg arg(const Char* name, const T& arg) { +inline auto arg(const Char* name, const T& arg) -> detail::named_arg { static_assert(!detail::is_named_arg(), "nested named arguments"); return {name, arg}; } /** \rst - A dynamic version of `fmt::format_arg_store`. - It's equipped with a storage to potentially temporary objects which lifetimes - could be shorter than the format arguments object. + A view of a collection of formatting arguments. To avoid lifetime issues it + should only be used as a parameter type in type-erased functions such as + ``vformat``:: - It can be implicitly converted into `~fmt::basic_format_args` for passing - into type-erased formatting functions such as `~fmt::vformat`. + void vlog(string_view format_str, format_args args); // OK + format_args args = make_format_args(42); // Error: dangling reference \endrst */ -template -class dynamic_format_arg_store -#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 - // Workaround a GCC template argument substitution bug. - : public basic_format_args -#endif -{ - private: - using char_type = typename Context::char_type; +template class basic_format_args { + public: + using size_type = int; + using format_arg = basic_format_arg; - template struct need_copy { - static constexpr detail::type mapped_type = - detail::mapped_type_constant::value; - - enum { - value = !(detail::is_reference_wrapper::value || - std::is_same>::value || - std::is_same>::value || - (mapped_type != detail::type::cstring_type && - mapped_type != detail::type::string_type && - mapped_type != detail::type::custom_type)) - }; + private: + // A descriptor that contains information about formatting arguments. + // If the number of arguments is less or equal to max_packed_args then + // argument types are passed in the descriptor. This reduces binary code size + // per formatting function call. + unsigned long long desc_; + union { + // If is_packed() returns true then argument values are stored in values_; + // otherwise they are stored in args_. This is done to improve cache + // locality and reduce compiled code size since storing larger objects + // may require more code (at least on x86-64) even if the same amount of + // data is actually copied to stack. It saves ~10% on the bloat test. + const detail::value* values_; + const format_arg* args_; }; - template - using stored_type = conditional_t::value, - std::basic_string, T>; - - // Storage of basic_format_arg must be contiguous. - std::vector> data_; - std::vector> named_info_; - - // Storage of arguments not fitting into basic_format_arg must grow - // without relocation because items in data_ refer to it. - detail::dynamic_arg_list dynamic_args_; - - friend class basic_format_args; - - unsigned long long get_types() const { - return detail::is_unpacked_bit | data_.size() | - (named_info_.empty() - ? 0ULL - : static_cast(detail::has_named_args_bit)); + constexpr auto is_packed() const -> bool { + return (desc_ & detail::is_unpacked_bit) == 0; } - - const basic_format_arg* data() const { - return named_info_.empty() ? data_.data() : data_.data() + 1; + auto has_named_args() const -> bool { + return (desc_ & detail::has_named_args_bit) != 0; } - template void emplace_arg(const T& arg) { - data_.emplace_back(detail::make_arg(arg)); + FMT_CONSTEXPR auto type(int index) const -> detail::type { + int shift = index * detail::packed_arg_bits; + unsigned int mask = (1 << detail::packed_arg_bits) - 1; + return static_cast((desc_ >> shift) & mask); } - template - void emplace_arg(const detail::named_arg& arg) { - if (named_info_.empty()) { - constexpr const detail::named_arg_info* zero_ptr{nullptr}; - data_.insert(data_.begin(), {zero_ptr, 0}); - } - data_.emplace_back(detail::make_arg(detail::unwrap(arg.value))); - auto pop_one = [](std::vector>* data) { - data->pop_back(); - }; - std::unique_ptr>, decltype(pop_one)> - guard{&data_, pop_one}; - named_info_.push_back({arg.name, static_cast(data_.size() - 2u)}); - data_[0].value_.named_args = {named_info_.data(), named_info_.size()}; - guard.release(); - } + constexpr FMT_INLINE basic_format_args(unsigned long long desc, + const detail::value* values) + : desc_(desc), values_(values) {} + constexpr basic_format_args(unsigned long long desc, const format_arg* args) + : desc_(desc), args_(args) {} public: - /** - \rst - Adds an argument into the dynamic store for later passing to a formatting - function. - - Note that custom types and string types (but not string views) are copied - into the store dynamically allocating memory if necessary. - - **Example**:: - - fmt::dynamic_format_arg_store store; - store.push_back(42); - store.push_back("abc"); - store.push_back(1.5f); - std::string result = fmt::vformat("{} and {} and {}", store); - \endrst - */ - template void push_back(const T& arg) { - if (detail::const_check(need_copy::value)) - emplace_arg(dynamic_args_.push>(arg)); - else - emplace_arg(detail::unwrap(arg)); - } - - /** - \rst - Adds a reference to the argument into the dynamic store for later passing to - a formatting function. Supports named arguments wrapped in - ``std::reference_wrapper`` via ``std::ref()``/``std::cref()``. - - **Example**:: - - fmt::dynamic_format_arg_store store; - char str[] = "1234567890"; - store.push_back(std::cref(str)); - int a1_val{42}; - auto a1 = fmt::arg("a1_", a1_val); - store.push_back(std::cref(a1)); - - // Changing str affects the output but only for string and custom types. - str[0] = 'X'; - - std::string result = fmt::vformat("{} and {a1_}"); - assert(result == "X234567890 and 42"); - \endrst - */ - template void push_back(std::reference_wrapper arg) { - static_assert( - detail::is_named_arg::type>::value || - need_copy::value, - "objects of built-in types and string views are always copied"); - emplace_arg(arg.get()); - } - - /** - Adds named argument into the dynamic store for later passing to a formatting - function. ``std::reference_wrapper`` is supported to avoid copying of the - argument. - */ - template - void push_back(const detail::named_arg& arg) { - const char_type* arg_name = - dynamic_args_.push>(arg.name).c_str(); - if (detail::const_check(need_copy::value)) { - emplace_arg( - fmt::arg(arg_name, dynamic_args_.push>(arg.value))); - } else { - emplace_arg(fmt::arg(arg_name, arg.value)); - } - } - - /** Erase all elements from the store */ - void clear() { - data_.clear(); - named_info_.clear(); - dynamic_args_ = detail::dynamic_arg_list(); - } - - /** - \rst - Reserves space to store at least *new_cap* arguments including - *new_cap_named* named arguments. - \endrst - */ - void reserve(size_t new_cap, size_t new_cap_named) { - FMT_ASSERT(new_cap >= new_cap_named, - "Set of arguments includes set of named arguments"); - data_.reserve(new_cap); - named_info_.reserve(new_cap_named); - } -}; - -/** - \rst - A view of a collection of formatting arguments. To avoid lifetime issues it - should only be used as a parameter type in type-erased functions such as - ``vformat``:: - - void vlog(string_view format_str, format_args args); // OK - format_args args = make_format_args(42); // Error: dangling reference - \endrst - */ -template class basic_format_args { - public: - using size_type = int; - using format_arg = basic_format_arg; - - private: - // A descriptor that contains information about formatting arguments. - // If the number of arguments is less or equal to max_packed_args then - // argument types are passed in the descriptor. This reduces binary code size - // per formatting function call. - unsigned long long desc_; - union { - // If is_packed() returns true then argument values are stored in values_; - // otherwise they are stored in args_. This is done to improve cache - // locality and reduce compiled code size since storing larger objects - // may require more code (at least on x86-64) even if the same amount of - // data is actually copied to stack. It saves ~10% on the bloat test. - const detail::value* values_; - const format_arg* args_; - }; - - bool is_packed() const { return (desc_ & detail::is_unpacked_bit) == 0; } - bool has_named_args() const { - return (desc_ & detail::has_named_args_bit) != 0; - } - - detail::type type(int index) const { - int shift = index * detail::packed_arg_bits; - unsigned int mask = (1 << detail::packed_arg_bits) - 1; - return static_cast((desc_ >> shift) & mask); - } - - basic_format_args(unsigned long long desc, - const detail::value* values) - : desc_(desc), values_(values) {} - basic_format_args(unsigned long long desc, const format_arg* args) - : desc_(desc), args_(args) {} - - public: - basic_format_args() : desc_(0) {} + constexpr basic_format_args() : desc_(0), args_(nullptr) {} /** \rst @@ -1872,8 +1786,10 @@ \endrst */ template - FMT_INLINE basic_format_args(const format_arg_store& store) - : basic_format_args(store.desc, store.data_.args()) {} + constexpr FMT_INLINE basic_format_args( + const format_arg_store& store) + : basic_format_args(format_arg_store::desc, + store.data_.args()) {} /** \rst @@ -1881,7 +1797,8 @@ `~fmt::dynamic_format_arg_store`. \endrst */ - FMT_INLINE basic_format_args(const dynamic_format_arg_store& store) + constexpr FMT_INLINE basic_format_args( + const dynamic_format_arg_store& store) : basic_format_args(store.get_types(), store.data()) {} /** @@ -1889,12 +1806,12 @@ Constructs a `basic_format_args` object from a dynamic set of arguments. \endrst */ - basic_format_args(const format_arg* args, int count) + constexpr basic_format_args(const format_arg* args, int count) : basic_format_args(detail::is_unpacked_bit | detail::to_unsigned(count), args) {} /** Returns the argument with the specified id. */ - format_arg get(int id) const { + FMT_CONSTEXPR auto get(int id) const -> format_arg { format_arg arg; if (!is_packed()) { if (id < max_size()) arg = args_[id]; @@ -1907,12 +1824,14 @@ return arg; } - template format_arg get(basic_string_view name) const { + template + auto get(basic_string_view name) const -> format_arg { int id = get_id(name); return id >= 0 ? get(id) : format_arg(); } - template int get_id(basic_string_view name) const { + template + auto get_id(basic_string_view name) const -> int { if (!has_named_args()) return -1; const auto& named_args = (is_packed() ? values_[-1] : args_[-1].value_).named_args; @@ -1922,87 +1841,1078 @@ return -1; } - int max_size() const { + auto max_size() const -> int { unsigned long long max_packed = detail::max_packed_args; return static_cast(is_packed() ? max_packed : desc_ & ~detail::is_unpacked_bit); } }; -#ifdef FMT_ARM_ABI_COMPATIBILITY /** An alias to ``basic_format_args``. */ -// Separate types would result in shorter symbols but break ABI compatibility +// A separate type would result in shorter symbols but break ABI compatibility // between clang and gcc on ARM (#1919). using format_args = basic_format_args; -using wformat_args = basic_format_args; -#else -// DEPRECATED! These are kept for ABI compatibility. -// It is a separate type rather than an alias to make symbols readable. -struct format_args : basic_format_args { - template - FMT_INLINE format_args(const Args&... args) : basic_format_args(args...) {} + +// We cannot use enum classes as bit fields because of a gcc bug +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414. +namespace align { +enum type { none, left, right, center, numeric }; +} +using align_t = align::type; +namespace sign { +enum type { none, minus, plus, space }; +} +using sign_t = sign::type; + +FMT_BEGIN_DETAIL_NAMESPACE + +void throw_format_error(const char* message); + +// Workaround an array initialization issue in gcc 4.8. +template struct fill_t { + private: + enum { max_size = 4 }; + Char data_[max_size] = {Char(' '), Char(0), Char(0), Char(0)}; + unsigned char size_ = 1; + + public: + FMT_CONSTEXPR void operator=(basic_string_view s) { + auto size = s.size(); + if (size > max_size) return throw_format_error("invalid fill"); + for (size_t i = 0; i < size; ++i) data_[i] = s[i]; + size_ = static_cast(size); + } + + constexpr auto size() const -> size_t { return size_; } + constexpr auto data() const -> const Char* { return data_; } + + FMT_CONSTEXPR auto operator[](size_t index) -> Char& { return data_[index]; } + FMT_CONSTEXPR auto operator[](size_t index) const -> const Char& { + return data_[index]; + } +}; +FMT_END_DETAIL_NAMESPACE + +// Format specifiers for built-in and string types. +template struct basic_format_specs { + int width; + int precision; + char type; + align_t align : 4; + sign_t sign : 3; + bool alt : 1; // Alternate form ('#'). + bool localized : 1; + detail::fill_t fill; + + constexpr basic_format_specs() + : width(0), + precision(-1), + type(0), + align(align::none), + sign(sign::none), + alt(false), + localized(false) {} +}; + +using format_specs = basic_format_specs; + +FMT_BEGIN_DETAIL_NAMESPACE + +enum class arg_id_kind { none, index, name }; + +// An argument reference. +template struct arg_ref { + FMT_CONSTEXPR arg_ref() : kind(arg_id_kind::none), val() {} + + FMT_CONSTEXPR explicit arg_ref(int index) + : kind(arg_id_kind::index), val(index) {} + FMT_CONSTEXPR explicit arg_ref(basic_string_view name) + : kind(arg_id_kind::name), val(name) {} + + FMT_CONSTEXPR auto operator=(int idx) -> arg_ref& { + kind = arg_id_kind::index; + val.index = idx; + return *this; + } + + arg_id_kind kind; + union value { + FMT_CONSTEXPR value(int id = 0) : index{id} {} + FMT_CONSTEXPR value(basic_string_view n) : name(n) {} + + int index; + basic_string_view name; + } val; }; -struct wformat_args : basic_format_args { - using basic_format_args::basic_format_args; + +// Format specifiers with width and precision resolved at formatting rather +// than parsing time to allow re-using the same parsed specifiers with +// different sets of arguments (precompilation of format strings). +template +struct dynamic_format_specs : basic_format_specs { + arg_ref width_ref; + arg_ref precision_ref; }; + +struct auto_id {}; + +// A format specifier handler that sets fields in basic_format_specs. +template class specs_setter { + protected: + basic_format_specs& specs_; + + public: + explicit FMT_CONSTEXPR specs_setter(basic_format_specs& specs) + : specs_(specs) {} + + FMT_CONSTEXPR specs_setter(const specs_setter& other) + : specs_(other.specs_) {} + + FMT_CONSTEXPR void on_align(align_t align) { specs_.align = align; } + FMT_CONSTEXPR void on_fill(basic_string_view fill) { + specs_.fill = fill; + } + FMT_CONSTEXPR void on_sign(sign_t s) { specs_.sign = s; } + FMT_CONSTEXPR void on_hash() { specs_.alt = true; } + FMT_CONSTEXPR void on_localized() { specs_.localized = true; } + + FMT_CONSTEXPR void on_zero() { + if (specs_.align == align::none) specs_.align = align::numeric; + specs_.fill[0] = Char('0'); + } + + FMT_CONSTEXPR void on_width(int width) { specs_.width = width; } + FMT_CONSTEXPR void on_precision(int precision) { + specs_.precision = precision; + } + FMT_CONSTEXPR void end_precision() {} + + FMT_CONSTEXPR void on_type(Char type) { + specs_.type = static_cast(type); + } +}; + +// Format spec handler that saves references to arguments representing dynamic +// width and precision to be resolved at formatting time. +template +class dynamic_specs_handler + : public specs_setter { + public: + using char_type = typename ParseContext::char_type; + + FMT_CONSTEXPR dynamic_specs_handler(dynamic_format_specs& specs, + ParseContext& ctx) + : specs_setter(specs), specs_(specs), context_(ctx) {} + + FMT_CONSTEXPR dynamic_specs_handler(const dynamic_specs_handler& other) + : specs_setter(other), + specs_(other.specs_), + context_(other.context_) {} + + template FMT_CONSTEXPR void on_dynamic_width(Id arg_id) { + specs_.width_ref = make_arg_ref(arg_id); + } + + template FMT_CONSTEXPR void on_dynamic_precision(Id arg_id) { + specs_.precision_ref = make_arg_ref(arg_id); + } + + FMT_CONSTEXPR void on_error(const char* message) { + context_.on_error(message); + } + + private: + dynamic_format_specs& specs_; + ParseContext& context_; + + using arg_ref_type = arg_ref; + + FMT_CONSTEXPR auto make_arg_ref(int arg_id) -> arg_ref_type { + context_.check_arg_id(arg_id); + return arg_ref_type(arg_id); + } + + FMT_CONSTEXPR auto make_arg_ref(auto_id) -> arg_ref_type { + return arg_ref_type(context_.next_arg_id()); + } + + FMT_CONSTEXPR auto make_arg_ref(basic_string_view arg_id) + -> arg_ref_type { + context_.check_arg_id(arg_id); + basic_string_view format_str( + context_.begin(), to_unsigned(context_.end() - context_.begin())); + return arg_ref_type(arg_id); + } +}; + +template constexpr bool is_ascii_letter(Char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +// Converts a character to ASCII. Returns a number > 127 on conversion failure. +template ::value)> +constexpr auto to_ascii(Char value) -> Char { + return value; +} +template ::value)> +constexpr auto to_ascii(Char value) -> + typename std::underlying_type::type { + return value; +} + +template +FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int { + if (const_check(sizeof(Char) != 1)) return 1; + constexpr char lengths[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0}; + int len = lengths[static_cast(*begin) >> 3]; + + // Compute the pointer to the next character early so that the next + // iteration can start working on the next character. Neither Clang + // nor GCC figure out this reordering on their own. + return len + !len; +} + +// Return the result via the out param to workaround gcc bug 77539. +template +FMT_CONSTEXPR auto find(Ptr first, Ptr last, T value, Ptr& out) -> bool { + for (out = first; out != last; ++out) { + if (*out == value) return true; + } + return false; +} + +template <> +inline auto find(const char* first, const char* last, char value, + const char*& out) -> bool { + out = static_cast( + std::memchr(first, value, to_unsigned(last - first))); + return out != nullptr; +} + +// Parses the range [begin, end) as an unsigned integer. This function assumes +// that the range is non-empty and the first character is a digit. +template +FMT_CONSTEXPR auto parse_nonnegative_int(const Char*& begin, const Char* end, + int error_value) noexcept -> int { + FMT_ASSERT(begin != end && '0' <= *begin && *begin <= '9', ""); + unsigned value = 0, prev = 0; + auto p = begin; + do { + prev = value; + value = value * 10 + unsigned(*p - '0'); + ++p; + } while (p != end && '0' <= *p && *p <= '9'); + auto num_digits = p - begin; + begin = p; + if (num_digits <= std::numeric_limits::digits10) + return static_cast(value); + // Check for overflow. + const unsigned max = to_unsigned((std::numeric_limits::max)()); + return num_digits == std::numeric_limits::digits10 + 1 && + prev * 10ull + unsigned(p[-1] - '0') <= max + ? static_cast(value) + : error_value; +} + +// Parses fill and alignment. +template +FMT_CONSTEXPR auto parse_align(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + FMT_ASSERT(begin != end, ""); + auto align = align::none; + auto p = begin + code_point_length(begin); + if (p >= end) p = begin; + for (;;) { + switch (to_ascii(*p)) { + case '<': + align = align::left; + break; + case '>': + align = align::right; + break; + case '^': + align = align::center; + break; + default: + break; + } + if (align != align::none) { + if (p != begin) { + auto c = *begin; + if (c == '{') + return handler.on_error("invalid fill character '{'"), begin; + handler.on_fill(basic_string_view(begin, to_unsigned(p - begin))); + begin = p + 1; + } else + ++begin; + handler.on_align(align); + break; + } else if (p == begin) { + break; + } + p = begin; + } + return begin; +} + +template FMT_CONSTEXPR bool is_name_start(Char c) { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || '_' == c; +} + +template +FMT_CONSTEXPR auto do_parse_arg_id(const Char* begin, const Char* end, + IDHandler&& handler) -> const Char* { + FMT_ASSERT(begin != end, ""); + Char c = *begin; + if (c >= '0' && c <= '9') { + int index = 0; + if (c != '0') + index = + parse_nonnegative_int(begin, end, (std::numeric_limits::max)()); + else + ++begin; + if (begin == end || (*begin != '}' && *begin != ':')) + handler.on_error("invalid format string"); + else + handler(index); + return begin; + } + if (!is_name_start(c)) { + handler.on_error("invalid format string"); + return begin; + } + auto it = begin; + do { + ++it; + } while (it != end && (is_name_start(c = *it) || ('0' <= c && c <= '9'))); + handler(basic_string_view(begin, to_unsigned(it - begin))); + return it; +} + +template +FMT_CONSTEXPR FMT_INLINE auto parse_arg_id(const Char* begin, const Char* end, + IDHandler&& handler) -> const Char* { + Char c = *begin; + if (c != '}' && c != ':') return do_parse_arg_id(begin, end, handler); + handler(); + return begin; +} + +template +FMT_CONSTEXPR auto parse_width(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + using detail::auto_id; + struct width_adapter { + Handler& handler; + + FMT_CONSTEXPR void operator()() { handler.on_dynamic_width(auto_id()); } + FMT_CONSTEXPR void operator()(int id) { handler.on_dynamic_width(id); } + FMT_CONSTEXPR void operator()(basic_string_view id) { + handler.on_dynamic_width(id); + } + FMT_CONSTEXPR void on_error(const char* message) { + if (message) handler.on_error(message); + } + }; + + FMT_ASSERT(begin != end, ""); + if ('0' <= *begin && *begin <= '9') { + int width = parse_nonnegative_int(begin, end, -1); + if (width != -1) + handler.on_width(width); + else + handler.on_error("number is too big"); + } else if (*begin == '{') { + ++begin; + if (begin != end) begin = parse_arg_id(begin, end, width_adapter{handler}); + if (begin == end || *begin != '}') + return handler.on_error("invalid format string"), begin; + ++begin; + } + return begin; +} + +template +FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + using detail::auto_id; + struct precision_adapter { + Handler& handler; + + FMT_CONSTEXPR void operator()() { handler.on_dynamic_precision(auto_id()); } + FMT_CONSTEXPR void operator()(int id) { handler.on_dynamic_precision(id); } + FMT_CONSTEXPR void operator()(basic_string_view id) { + handler.on_dynamic_precision(id); + } + FMT_CONSTEXPR void on_error(const char* message) { + if (message) handler.on_error(message); + } + }; + + ++begin; + auto c = begin != end ? *begin : Char(); + if ('0' <= c && c <= '9') { + auto precision = parse_nonnegative_int(begin, end, -1); + if (precision != -1) + handler.on_precision(precision); + else + handler.on_error("number is too big"); + } else if (c == '{') { + ++begin; + if (begin != end) + begin = parse_arg_id(begin, end, precision_adapter{handler}); + if (begin == end || *begin++ != '}') + return handler.on_error("invalid format string"), begin; + } else { + return handler.on_error("missing precision specifier"), begin; + } + handler.end_precision(); + return begin; +} + +// Parses standard format specifiers and sends notifications about parsed +// components to handler. +template +FMT_CONSTEXPR FMT_INLINE auto parse_format_specs(const Char* begin, + const Char* end, + SpecHandler&& handler) + -> const Char* { + if (begin + 1 < end && begin[1] == '}' && is_ascii_letter(*begin) && + *begin != 'L') { + handler.on_type(*begin++); + return begin; + } + + if (begin == end) return begin; + + begin = parse_align(begin, end, handler); + if (begin == end) return begin; + + // Parse sign. + switch (to_ascii(*begin)) { + case '+': + handler.on_sign(sign::plus); + ++begin; + break; + case '-': + handler.on_sign(sign::minus); + ++begin; + break; + case ' ': + handler.on_sign(sign::space); + ++begin; + break; + default: + break; + } + if (begin == end) return begin; + + if (*begin == '#') { + handler.on_hash(); + if (++begin == end) return begin; + } + + // Parse zero flag. + if (*begin == '0') { + handler.on_zero(); + if (++begin == end) return begin; + } + + begin = parse_width(begin, end, handler); + if (begin == end) return begin; + + // Parse precision. + if (*begin == '.') { + begin = parse_precision(begin, end, handler); + if (begin == end) return begin; + } + + if (*begin == 'L') { + handler.on_localized(); + ++begin; + } + + // Parse type. + if (begin != end && *begin != '}') handler.on_type(*begin++); + return begin; +} + +template +FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + struct id_adapter { + Handler& handler; + int arg_id; + + FMT_CONSTEXPR void operator()() { arg_id = handler.on_arg_id(); } + FMT_CONSTEXPR void operator()(int id) { arg_id = handler.on_arg_id(id); } + FMT_CONSTEXPR void operator()(basic_string_view id) { + arg_id = handler.on_arg_id(id); + } + FMT_CONSTEXPR void on_error(const char* message) { + if (message) handler.on_error(message); + } + }; + + ++begin; + if (begin == end) return handler.on_error("invalid format string"), end; + if (*begin == '}') { + handler.on_replacement_field(handler.on_arg_id(), begin); + } else if (*begin == '{') { + handler.on_text(begin, begin + 1); + } else { + auto adapter = id_adapter{handler, 0}; + begin = parse_arg_id(begin, end, adapter); + Char c = begin != end ? *begin : Char(); + if (c == '}') { + handler.on_replacement_field(adapter.arg_id, begin); + } else if (c == ':') { + begin = handler.on_format_specs(adapter.arg_id, begin + 1, end); + if (begin == end || *begin != '}') + return handler.on_error("unknown format specifier"), end; + } else { + return handler.on_error("missing '}' in format string"), end; + } + } + return begin + 1; +} + +template +FMT_CONSTEXPR FMT_INLINE void parse_format_string( + basic_string_view format_str, Handler&& handler) { + // this is most likely a name-lookup defect in msvc's modules implementation + using detail::find; + + auto begin = format_str.data(); + auto end = begin + format_str.size(); + if (end - begin < 32) { + // Use a simple loop instead of memchr for small strings. + const Char* p = begin; + while (p != end) { + auto c = *p++; + if (c == '{') { + handler.on_text(begin, p - 1); + begin = p = parse_replacement_field(p - 1, end, handler); + } else if (c == '}') { + if (p == end || *p != '}') + return handler.on_error("unmatched '}' in format string"); + handler.on_text(begin, p); + begin = ++p; + } + } + handler.on_text(begin, end); + return; + } + struct writer { + FMT_CONSTEXPR void operator()(const Char* pbegin, const Char* pend) { + if (pbegin == pend) return; + for (;;) { + const Char* p = nullptr; + if (!find(pbegin, pend, Char('}'), p)) + return handler_.on_text(pbegin, pend); + ++p; + if (p == pend || *p != '}') + return handler_.on_error("unmatched '}' in format string"); + handler_.on_text(pbegin, p); + pbegin = p + 1; + } + } + Handler& handler_; + } write{handler}; + while (begin != end) { + // Doing two passes with memchr (one for '{' and another for '}') is up to + // 2.5x faster than the naive one-pass implementation on big format strings. + const Char* p = begin; + if (*begin != '{' && !find(begin + 1, end, Char('{'), p)) + return write(begin, end); + write(begin, p); + begin = parse_replacement_field(p, end, handler); + } +} + +template +FMT_CONSTEXPR auto parse_format_specs(ParseContext& ctx) + -> decltype(ctx.begin()) { + using char_type = typename ParseContext::char_type; + using context = buffer_context; + using mapped_type = conditional_t< + mapped_type_constant::value != type::custom_type, + decltype(arg_mapper().map(std::declval())), T>; + auto f = conditional_t::value, + formatter, + fallback_formatter>(); + return f.parse(ctx); +} + +// A parse context with extra argument id checks. It is only used at compile +// time because adding checks at runtime would introduce substantial overhead +// and would be redundant since argument ids are checked when arguments are +// retrieved anyway. +template +class compile_parse_context + : public basic_format_parse_context { + private: + int num_args_; + using base = basic_format_parse_context; + + public: + explicit FMT_CONSTEXPR compile_parse_context( + basic_string_view format_str, + int num_args = (std::numeric_limits::max)(), ErrorHandler eh = {}) + : base(format_str, eh), num_args_(num_args) {} + + FMT_CONSTEXPR auto next_arg_id() -> int { + int id = base::next_arg_id(); + if (id >= num_args_) this->on_error("argument not found"); + return id; + } + + FMT_CONSTEXPR void check_arg_id(int id) { + base::check_arg_id(id); + if (id >= num_args_) this->on_error("argument not found"); + } + using base::check_arg_id; +}; + +template +FMT_CONSTEXPR void check_int_type_spec(char spec, ErrorHandler&& eh) { + switch (spec) { + case 0: + case 'd': + case 'x': + case 'X': + case 'b': + case 'B': + case 'o': + case 'c': + break; + default: + eh.on_error("invalid type specifier"); + break; + } +} + +// Checks char specs and returns true if the type spec is char (and not int). +template +FMT_CONSTEXPR auto check_char_specs(const basic_format_specs& specs, + ErrorHandler&& eh = {}) -> bool { + if (specs.type && specs.type != 'c') { + check_int_type_spec(specs.type, eh); + return false; + } + if (specs.align == align::numeric || specs.sign != sign::none || specs.alt) + eh.on_error("invalid format specifier for char"); + return true; +} + +// A floating-point presentation format. +enum class float_format : unsigned char { + general, // General: exponent notation or fixed point based on magnitude. + exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3. + fixed, // Fixed point with the default precision of 6, e.g. 0.0012. + hex +}; + +struct float_specs { + int precision; + float_format format : 8; + sign_t sign : 8; + bool upper : 1; + bool locale : 1; + bool binary32 : 1; + bool use_grisu : 1; + bool showpoint : 1; +}; + +template +FMT_CONSTEXPR auto parse_float_type_spec(const basic_format_specs& specs, + ErrorHandler&& eh = {}) + -> float_specs { + auto result = float_specs(); + result.showpoint = specs.alt; + result.locale = specs.localized; + switch (specs.type) { + case 0: + result.format = float_format::general; + break; + case 'G': + result.upper = true; + FMT_FALLTHROUGH; + case 'g': + result.format = float_format::general; + break; + case 'E': + result.upper = true; + FMT_FALLTHROUGH; + case 'e': + result.format = float_format::exp; + result.showpoint |= specs.precision != 0; + break; + case 'F': + result.upper = true; + FMT_FALLTHROUGH; + case 'f': + result.format = float_format::fixed; + result.showpoint |= specs.precision != 0; + break; + case 'A': + result.upper = true; + FMT_FALLTHROUGH; + case 'a': + result.format = float_format::hex; + break; + default: + eh.on_error("invalid type specifier"); + break; + } + return result; +} + +template +FMT_CONSTEXPR auto check_cstring_type_spec(Char spec, ErrorHandler&& eh = {}) + -> bool { + if (spec == 0 || spec == 's') return true; + if (spec != 'p') eh.on_error("invalid type specifier"); + return false; +} + +template +FMT_CONSTEXPR void check_string_type_spec(Char spec, ErrorHandler&& eh = {}) { + if (spec != 0 && spec != 's') eh.on_error("invalid type specifier"); +} + +template +FMT_CONSTEXPR void check_pointer_type_spec(Char spec, ErrorHandler&& eh) { + if (spec != 0 && spec != 'p') eh.on_error("invalid type specifier"); +} + +// A parse_format_specs handler that checks if specifiers are consistent with +// the argument type. +template class specs_checker : public Handler { + private: + detail::type arg_type_; + + FMT_CONSTEXPR void require_numeric_argument() { + if (!is_arithmetic_type(arg_type_)) + this->on_error("format specifier requires numeric argument"); + } + + public: + FMT_CONSTEXPR specs_checker(const Handler& handler, detail::type arg_type) + : Handler(handler), arg_type_(arg_type) {} + + FMT_CONSTEXPR void on_align(align_t align) { + if (align == align::numeric) require_numeric_argument(); + Handler::on_align(align); + } + + FMT_CONSTEXPR void on_sign(sign_t s) { + require_numeric_argument(); + if (is_integral_type(arg_type_) && arg_type_ != type::int_type && + arg_type_ != type::long_long_type && arg_type_ != type::char_type) { + this->on_error("format specifier requires signed argument"); + } + Handler::on_sign(s); + } + + FMT_CONSTEXPR void on_hash() { + require_numeric_argument(); + Handler::on_hash(); + } + + FMT_CONSTEXPR void on_localized() { + require_numeric_argument(); + Handler::on_localized(); + } + + FMT_CONSTEXPR void on_zero() { + require_numeric_argument(); + Handler::on_zero(); + } + + FMT_CONSTEXPR void end_precision() { + if (is_integral_type(arg_type_) || arg_type_ == type::pointer_type) + this->on_error("precision not allowed for this argument type"); + } +}; + +constexpr int invalid_arg_index = -1; + +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS +template +constexpr auto get_arg_index_by_name(basic_string_view name) -> int { + if constexpr (detail::is_statically_named_arg()) { + if (name == T::name) return N; + } + if constexpr (sizeof...(Args) > 0) { + return get_arg_index_by_name(name); + } else { + (void)name; // Workaround an MSVC bug about "unused" parameter. + return invalid_arg_index; + } +} +#endif + +template +FMT_CONSTEXPR auto get_arg_index_by_name(basic_string_view name) -> int { +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS + if constexpr (sizeof...(Args) > 0) { + return get_arg_index_by_name<0, Args...>(name); + } else { + (void)name; + return invalid_arg_index; + } +#else + (void)name; + return invalid_arg_index; #endif +} -namespace detail { +template +class format_string_checker { + private: + using parse_context_type = compile_parse_context; + enum { num_args = sizeof...(Args) }; -template ::value)> -std::basic_string vformat( - basic_string_view format_str, - basic_format_args>> args); + // Format specifier parsing function. + using parse_func = const Char* (*)(parse_context_type&); -FMT_API std::string vformat(string_view format_str, format_args args); + parse_context_type context_; + parse_func parse_funcs_[num_args > 0 ? num_args : 1]; + + public: + explicit FMT_CONSTEXPR format_string_checker( + basic_string_view format_str, ErrorHandler eh) + : context_(format_str, num_args, eh), + parse_funcs_{&parse_format_specs...} {} + + FMT_CONSTEXPR void on_text(const Char*, const Char*) {} + + FMT_CONSTEXPR auto on_arg_id() -> int { return context_.next_arg_id(); } + FMT_CONSTEXPR auto on_arg_id(int id) -> int { + return context_.check_arg_id(id), id; + } + FMT_CONSTEXPR auto on_arg_id(basic_string_view id) -> int { +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS + auto index = get_arg_index_by_name(id); + if (index == invalid_arg_index) on_error("named argument is not found"); + return context_.check_arg_id(index), index; +#else + (void)id; + on_error("compile-time checks for named arguments require C++20 support"); + return 0; +#endif + } + + FMT_CONSTEXPR void on_replacement_field(int, const Char*) {} + + FMT_CONSTEXPR auto on_format_specs(int id, const Char* begin, const Char*) + -> const Char* { + context_.advance_to(context_.begin() + (begin - &*context_.begin())); + // id >= 0 check is a workaround for gcc 10 bug (#2065). + return id >= 0 && id < num_args ? parse_funcs_[id](context_) : begin; + } + + FMT_CONSTEXPR void on_error(const char* message) { + context_.on_error(message); + } +}; + +template ::value), int>> +void check_format_string(S format_str) { + FMT_CONSTEXPR auto s = to_string_view(format_str); + using checker = format_string_checker...>; + FMT_CONSTEXPR bool invalid_format = + (parse_format_string(s, checker(s, {})), true); + ignore_unused(invalid_format); +} template void vformat_to( - buffer& buf, basic_string_view format_str, + buffer& buf, basic_string_view fmt, basic_format_args)> args, - detail::locale_ref loc = {}); - -template ::value)> -inline void vprint_mojibake(std::FILE*, basic_string_view, const Args&) {} + locale_ref loc = {}); FMT_API void vprint_mojibake(std::FILE*, string_view, format_args); #ifndef _WIN32 inline void vprint_mojibake(std::FILE*, string_view, format_args) {} #endif -} // namespace detail +FMT_END_DETAIL_NAMESPACE + +// A formatter specialization for the core types corresponding to detail::type +// constants. +template +struct formatter::value != + detail::type::custom_type>> { + private: + detail::dynamic_format_specs specs_; + + public: + // Parses format specifiers stopping either at the end of the range or at the + // terminating '}'. + template + FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) { + auto begin = ctx.begin(), end = ctx.end(); + if (begin == end) return begin; + using handler_type = detail::dynamic_specs_handler; + auto type = detail::type_constant::value; + auto checker = + detail::specs_checker(handler_type(specs_, ctx), type); + auto it = detail::parse_format_specs(begin, end, checker); + auto eh = ctx.error_handler(); + switch (type) { + case detail::type::none_type: + FMT_ASSERT(false, "invalid argument type"); + break; + case detail::type::bool_type: + if (!specs_.type || specs_.type == 's') break; + FMT_FALLTHROUGH; + case detail::type::int_type: + case detail::type::uint_type: + case detail::type::long_long_type: + case detail::type::ulong_long_type: + case detail::type::int128_type: + case detail::type::uint128_type: + detail::check_int_type_spec(specs_.type, eh); + break; + case detail::type::char_type: + detail::check_char_specs(specs_, eh); + break; + case detail::type::float_type: + if (detail::const_check(FMT_USE_FLOAT)) + detail::parse_float_type_spec(specs_, eh); + else + FMT_ASSERT(false, "float support disabled"); + break; + case detail::type::double_type: + if (detail::const_check(FMT_USE_DOUBLE)) + detail::parse_float_type_spec(specs_, eh); + else + FMT_ASSERT(false, "double support disabled"); + break; + case detail::type::long_double_type: + if (detail::const_check(FMT_USE_LONG_DOUBLE)) + detail::parse_float_type_spec(specs_, eh); + else + FMT_ASSERT(false, "long double support disabled"); + break; + case detail::type::cstring_type: + detail::check_cstring_type_spec(specs_.type, eh); + break; + case detail::type::string_type: + detail::check_string_type_spec(specs_.type, eh); + break; + case detail::type::pointer_type: + detail::check_pointer_type_spec(specs_.type, eh); + break; + case detail::type::custom_type: + // Custom format specifiers are checked in parse functions of + // formatter specializations. + break; + } + return it; + } + + template + FMT_CONSTEXPR auto format(const T& val, FormatContext& ctx) const + -> decltype(ctx.out()); +}; + +template struct basic_runtime { basic_string_view str; }; + +template class basic_format_string { + private: + basic_string_view str_; + + public: + template >::value)> + FMT_CONSTEVAL basic_format_string(const S& s) : str_(s) { + static_assert( + detail::count< + (std::is_base_of>::value && + std::is_reference::value)...>() == 0, + "passing views as lvalues is disallowed"); +#ifdef FMT_HAS_CONSTEVAL + if constexpr (detail::count_named_args() == 0) { + using checker = detail::format_string_checker...>; + detail::parse_format_string(str_, checker(s, {})); + } +#else + detail::check_format_string(s); +#endif + } + basic_format_string(basic_runtime r) : str_(r.str) {} + + FMT_INLINE operator basic_string_view() const { return str_; } +}; + +#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 +// Workaround broken conversion on older gcc. +template using format_string = string_view; +template auto runtime(const S& s) -> basic_string_view> { + return s; +} +#else +template +using format_string = basic_format_string...>; +// Creates a runtime format string. +template auto runtime(const S& s) -> basic_runtime> { + return {{s}}; +} +#endif + +FMT_API auto vformat(string_view fmt, format_args args) -> std::string; + +/** + \rst + Formats ``args`` according to specifications in ``fmt`` and returns the result + as a string. + + **Example**:: + + #include + std::string message = fmt::format("The answer is {}", 42); + \endrst +*/ +template +FMT_INLINE auto format(format_string fmt, T&&... args) -> std::string { + return vformat(fmt, fmt::make_format_args(args...)); +} /** Formats a string and writes the output to ``out``. */ -// GCC 8 and earlier cannot handle std::back_insert_iterator with -// vformat_to(...) overload, so SFINAE on iterator type instead. -template , - bool enable = detail::is_output_iterator::value> -auto vformat_to(OutputIt out, const S& format_str, - basic_format_args>> args) - -> typename std::enable_if::type { - decltype(detail::get_buffer(out)) buf(detail::get_buffer_init(out)); - detail::vformat_to(buf, to_string_view(format_str), args); +template ::value)> +auto vformat_to(OutputIt out, string_view fmt, format_args args) -> OutputIt { + using detail::get_buffer; + auto&& buf = get_buffer(out); + detail::vformat_to(buf, string_view(fmt), args, {}); return detail::get_iterator(buf); } /** \rst - Formats arguments, writes the result to the output iterator ``out`` and returns - the iterator past the end of the output range. + Formats ``args`` according to specifications in ``fmt``, writes the result to + the output iterator ``out`` and returns the iterator past the end of the output + range. **Example**:: - std::vector out; + auto out = std::vector(); fmt::format_to(std::back_inserter(out), "{}", 42); \endrst */ -// We cannot use FMT_ENABLE_IF because of a bug in gcc 8.3. -template >::value> -inline auto format_to(OutputIt out, const S& format_str, Args&&... args) -> - typename std::enable_if::type { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return vformat_to(out, to_string_view(format_str), vargs); +template ::value)> +FMT_INLINE auto format_to(OutputIt out, format_string fmt, T&&... args) + -> OutputIt { + return vformat_to(out, fmt, fmt::make_format_args(args...)); } template struct format_to_n_result { @@ -2012,111 +2922,81 @@ size_t size; }; -template ::value)> -inline format_to_n_result vformat_to_n( - OutputIt out, size_t n, basic_string_view format_str, - basic_format_args>> args) { - detail::iterator_buffer buf(out, - n); - detail::vformat_to(buf, format_str, args); +template ::value)> +auto vformat_to_n(OutputIt out, size_t n, string_view fmt, format_args args) + -> format_to_n_result { + using buffer = + detail::iterator_buffer; + auto buf = buffer(out, n); + detail::vformat_to(buf, fmt, args, {}); return {buf.out(), buf.count()}; } /** - \rst - Formats arguments, writes up to ``n`` characters of the result to the output - iterator ``out`` and returns the total output size and the iterator past the - end of the output range. - \endrst + \rst + Formats ``args`` according to specifications in ``fmt``, writes up to ``n`` + characters of the result to the output iterator ``out`` and returns the total + (not truncated) output size and the iterator past the end of the output range. + \endrst */ -template >::value> -inline auto format_to_n(OutputIt out, size_t n, const S& format_str, - const Args&... args) -> - typename std::enable_if>::type { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return vformat_to_n(out, n, to_string_view(format_str), vargs); +template ::value)> +FMT_INLINE auto format_to_n(OutputIt out, size_t n, format_string fmt, + const T&... args) -> format_to_n_result { + return vformat_to_n(out, n, fmt, fmt::make_format_args(args...)); } -/** - Returns the number of characters in the output of - ``format(format_str, args...)``. - */ -template -inline size_t formatted_size(string_view format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - detail::counting_buffer<> buf; - detail::vformat_to(buf, format_str, vargs); +/** Returns the number of chars in the output of ``format(fmt, args...)``. */ +template +FMT_INLINE auto formatted_size(format_string fmt, T&&... args) -> size_t { + auto buf = detail::counting_buffer<>(); + detail::vformat_to(buf, string_view(fmt), fmt::make_format_args(args...), {}); return buf.count(); } -template > -FMT_INLINE std::basic_string vformat( - const S& format_str, - basic_format_args>> args) { - return detail::vformat(to_string_view(format_str), args); -} +FMT_API void vprint(string_view fmt, format_args args); +FMT_API void vprint(std::FILE* f, string_view fmt, format_args args); /** \rst - Formats arguments and returns the result as a string. + Formats ``args`` according to specifications in ``fmt`` and writes the output + to ``stdout``. **Example**:: - #include - std::string message = fmt::format("The answer is {}", 42); + fmt::print("Elapsed time: {0:.2f} seconds", 1.23); \endrst -*/ -// Pass char_t as a default template parameter instead of using -// std::basic_string> to reduce the symbol size. -template > -FMT_INLINE std::basic_string format(const S& format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return detail::vformat(to_string_view(format_str), vargs); + */ +template +FMT_INLINE void print(format_string fmt, T&&... args) { + const auto& vargs = fmt::make_format_args(args...); + return detail::is_utf8() ? vprint(fmt, vargs) + : detail::vprint_mojibake(stdout, fmt, vargs); } -FMT_API void vprint(string_view, format_args); -FMT_API void vprint(std::FILE*, string_view, format_args); - /** \rst - Formats ``args`` according to specifications in ``format_str`` and writes the - output to the file ``f``. Strings are assumed to be Unicode-encoded unless the - ``FMT_UNICODE`` macro is set to 0. + Formats ``args`` according to specifications in ``fmt`` and writes the + output to the file ``f``. **Example**:: fmt::print(stderr, "Don't {}!", "panic"); \endrst */ -template > -inline void print(std::FILE* f, const S& format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return detail::is_unicode() - ? vprint(f, to_string_view(format_str), vargs) - : detail::vprint_mojibake(f, to_string_view(format_str), vargs); +template +FMT_INLINE void print(std::FILE* f, format_string fmt, T&&... args) { + const auto& vargs = fmt::make_format_args(args...); + return detail::is_utf8() ? vprint(f, fmt, vargs) + : detail::vprint_mojibake(f, fmt, vargs); } -/** - \rst - Formats ``args`` according to specifications in ``format_str`` and writes - the output to ``stdout``. Strings are assumed to be Unicode-encoded unless - the ``FMT_UNICODE`` macro is set to 0. - - **Example**:: - - fmt::print("Elapsed time: {0:.2f} seconds", 1.23); - \endrst - */ -template > -inline void print(const S& format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return detail::is_unicode() - ? vprint(to_string_view(format_str), vargs) - : detail::vprint_mojibake(stdout, to_string_view(format_str), - vargs); -} +FMT_MODULE_EXPORT_END +FMT_GCC_PRAGMA("GCC pop_options") FMT_END_NAMESPACE +#ifdef FMT_HEADER_ONLY +# include "format.h" +#endif #endif // FMT_CORE_H_ diff -Nru ccache-4.2.1/src/third_party/fmt/format.h ccache-4.5.1/src/third_party/fmt/format.h --- ccache-4.2.1/src/third_party/fmt/format.h 2021-03-27 18:54:20.000000000 +0000 +++ ccache-4.5.1/src/third_party/fmt/format.h 2021-11-17 19:31:58.000000000 +0000 @@ -33,13 +33,13 @@ #ifndef FMT_FORMAT_H_ #define FMT_FORMAT_H_ -#include -#include -#include -#include -#include -#include -#include +#include // std::signbit +#include // uint32_t +#include // std::numeric_limits +#include // std::uninitialized_copy +#include // std::runtime_error +#include // std::system_error +#include // std::swap #include "core.h" @@ -69,30 +69,10 @@ # define FMT_NOINLINE #endif -#if __cplusplus == 201103L || __cplusplus == 201402L -# if defined(__INTEL_COMPILER) || defined(__PGI) -# define FMT_FALLTHROUGH -# elif defined(__clang__) -# define FMT_FALLTHROUGH [[clang::fallthrough]] -# elif FMT_GCC_VERSION >= 700 && \ - (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) -# define FMT_FALLTHROUGH [[gnu::fallthrough]] -# else -# define FMT_FALLTHROUGH -# endif -#elif FMT_HAS_CPP17_ATTRIBUTE(fallthrough) || \ - (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) -# define FMT_FALLTHROUGH [[fallthrough]] +#if FMT_MSC_VER +# define FMT_MSC_DEFAULT = default #else -# define FMT_FALLTHROUGH -#endif - -#ifndef FMT_MAYBE_UNUSED -# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused) -# define FMT_MAYBE_UNUSED [[maybe_unused]] -# else -# define FMT_MAYBE_UNUSED -# endif +# define FMT_MSC_DEFAULT #endif #ifndef FMT_THROW @@ -113,10 +93,9 @@ # define FMT_THROW(x) throw x # endif # else -# define FMT_THROW(x) \ - do { \ - static_cast(sizeof(x)); \ - FMT_ASSERT(false, ""); \ +# define FMT_THROW(x) \ + do { \ + FMT_ASSERT(false, (x).what()); \ } while (false) # endif #endif @@ -129,6 +108,27 @@ # define FMT_CATCH(x) if (false) #endif +#ifndef FMT_DEPRECATED +# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900 +# define FMT_DEPRECATED [[deprecated]] +# else +# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__) +# define FMT_DEPRECATED __attribute__((deprecated)) +# elif FMT_MSC_VER +# define FMT_DEPRECATED __declspec(deprecated) +# else +# define FMT_DEPRECATED /* deprecated */ +# endif +# endif +#endif + +// Workaround broken [[deprecated]] in the Intel, PGI and NVCC compilers. +#if FMT_ICC_VERSION || defined(__PGI) || FMT_NVCC +# define FMT_DEPRECATED_ALIAS +#else +# define FMT_DEPRECATED_ALIAS FMT_DEPRECATED +#endif + #ifndef FMT_USE_USER_DEFINED_LITERALS // EDG based compilers (Intel, NVIDIA, Elbrus, etc), GCC and MSVC support UDLs. # if (FMT_HAS_FEATURE(cxx_user_literals) || FMT_GCC_VERSION >= 407 || \ @@ -140,36 +140,10 @@ # endif #endif -#ifndef FMT_USE_UDL_TEMPLATE -// EDG frontend based compilers (icc, nvcc, PGI, etc) and GCC < 6.4 do not -// properly support UDL templates and GCC >= 9 warns about them. -# if FMT_USE_USER_DEFINED_LITERALS && \ - (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 501) && \ - ((FMT_GCC_VERSION >= 604 && __cplusplus >= 201402L) || \ - FMT_CLANG_VERSION >= 304) && \ - !defined(__PGI) && !defined(__NVCC__) -# define FMT_USE_UDL_TEMPLATE 1 -# else -# define FMT_USE_UDL_TEMPLATE 0 -# endif -#endif - -#ifndef FMT_USE_FLOAT -# define FMT_USE_FLOAT 1 -#endif - -#ifndef FMT_USE_DOUBLE -# define FMT_USE_DOUBLE 1 -#endif - -#ifndef FMT_USE_LONG_DOUBLE -# define FMT_USE_LONG_DOUBLE 1 -#endif - // Defining FMT_REDUCE_INT_INSTANTIATIONS to 1, will reduce the number of -// int_writer template instances to just one by only using the largest integer -// type. This results in a reduction in binary size but will cause a decrease in -// integer formatting performance. +// integer formatter template instantiations to just one by only using the +// largest integer type. This results in a reduction in binary size but will +// cause a decrease in integer formatting performance. #if !defined(FMT_REDUCE_INT_INSTANTIATIONS) # define FMT_REDUCE_INT_INSTANTIATIONS 0 #endif @@ -196,33 +170,33 @@ // Some compilers masquerade as both MSVC and GCC-likes or otherwise support // __builtin_clz and __builtin_clzll, so only define FMT_BUILTIN_CLZ using the // MSVC intrinsics if the clz and clzll builtins are not available. -#if FMT_MSC_VER && !defined(FMT_BUILTIN_CLZLL) && \ - !defined(FMT_BUILTIN_CTZLL) && !defined(_MANAGED) +#if FMT_MSC_VER && !defined(FMT_BUILTIN_CLZLL) && !defined(FMT_BUILTIN_CTZLL) FMT_BEGIN_NAMESPACE namespace detail { // Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning. -# ifndef __clang__ +# if !defined(__clang__) +# pragma managed(push, off) # pragma intrinsic(_BitScanForward) # pragma intrinsic(_BitScanReverse) -# endif -# if defined(_WIN64) && !defined(__clang__) -# pragma intrinsic(_BitScanForward64) -# pragma intrinsic(_BitScanReverse64) +# if defined(_WIN64) +# pragma intrinsic(_BitScanForward64) +# pragma intrinsic(_BitScanReverse64) +# endif # endif -inline int clz(uint32_t x) { +inline auto clz(uint32_t x) -> int { unsigned long r = 0; _BitScanReverse(&r, x); FMT_ASSERT(x != 0, ""); // Static analysis complains about using uninitialized data // "r", but the only way that can happen is if "x" is 0, // which the callers guarantee to not happen. - FMT_SUPPRESS_MSC_WARNING(6102) + FMT_MSC_WARNING(suppress : 6102) return 31 ^ static_cast(r); } # define FMT_BUILTIN_CLZ(n) detail::clz(n) -inline int clzll(uint64_t x) { +inline auto clzll(uint64_t x) -> int { unsigned long r = 0; # ifdef _WIN64 _BitScanReverse64(&r, x); @@ -233,24 +207,24 @@ _BitScanReverse(&r, static_cast(x)); # endif FMT_ASSERT(x != 0, ""); - FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning. return 63 ^ static_cast(r); } # define FMT_BUILTIN_CLZLL(n) detail::clzll(n) -inline int ctz(uint32_t x) { +inline auto ctz(uint32_t x) -> int { unsigned long r = 0; _BitScanForward(&r, x); FMT_ASSERT(x != 0, ""); - FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning. return static_cast(r); } # define FMT_BUILTIN_CTZ(n) detail::ctz(n) -inline int ctzll(uint64_t x) { +inline auto ctzll(uint64_t x) -> int { unsigned long r = 0; FMT_ASSERT(x != 0, ""); - FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning. # ifdef _WIN64 _BitScanForward64(&r, x); # else @@ -263,30 +237,35 @@ return static_cast(r); } # define FMT_BUILTIN_CTZLL(n) detail::ctzll(n) +# if !defined(__clang__) +# pragma managed(pop) +# endif } // namespace detail FMT_END_NAMESPACE #endif -// Enable the deprecated numeric alignment. -#ifndef FMT_DEPRECATED_NUMERIC_ALIGN -# define FMT_DEPRECATED_NUMERIC_ALIGN 0 -#endif - FMT_BEGIN_NAMESPACE namespace detail { +#if __cplusplus >= 202002L || \ + (__cplusplus >= 201709L && FMT_GCC_VERSION >= 1002) +# define FMT_CONSTEXPR20 constexpr +#else +# define FMT_CONSTEXPR20 +#endif + // An equivalent of `*reinterpret_cast(&source)` that doesn't have // undefined behavior (e.g. due to type aliasing). // Example: uint64_t d = bit_cast(2.718); template -inline Dest bit_cast(const Source& source) { +inline auto bit_cast(const Source& source) -> Dest { static_assert(sizeof(Dest) == sizeof(Source), "size mismatch"); Dest dest; std::memcpy(&dest, &source, sizeof(dest)); return dest; } -inline bool is_big_endian() { +inline auto is_big_endian() -> bool { const auto u = 1u; struct bytes { char data[sizeof(u)]; @@ -309,26 +288,28 @@ }; #ifdef UINTPTR_MAX using uintptr_t = ::uintptr_t; -inline uintptr_t to_uintptr(const void* p) { return bit_cast(p); } +inline auto to_uintptr(const void* p) -> uintptr_t { + return bit_cast(p); +} #else using uintptr_t = fallback_uintptr; -inline fallback_uintptr to_uintptr(const void* p) { +inline auto to_uintptr(const void* p) -> fallback_uintptr { return fallback_uintptr(p); } #endif // Returns the largest possible value for type T. Same as // std::numeric_limits::max() but shorter and not affected by the max macro. -template constexpr T max_value() { +template constexpr auto max_value() -> T { return (std::numeric_limits::max)(); } -template constexpr int num_bits() { +template constexpr auto num_bits() -> int { return std::numeric_limits::digits; } // std::numeric_limits::digits may return 0 for 128-bit ints. -template <> constexpr int num_bits() { return 128; } -template <> constexpr int num_bits() { return 128; } -template <> constexpr int num_bits() { +template <> constexpr auto num_bits() -> int { return 128; } +template <> constexpr auto num_bits() -> int { return 128; } +template <> constexpr auto num_bits() -> int { return static_cast(sizeof(void*) * std::numeric_limits::digits); } @@ -346,31 +327,35 @@ template using sentinel_t = decltype(std::end(std::declval())); // A workaround for std::string not having mutable data() until C++17. -template inline Char* get_data(std::basic_string& s) { +template +inline auto get_data(std::basic_string& s) -> Char* { return &s[0]; } template -inline typename Container::value_type* get_data(Container& c) { +inline auto get_data(Container& c) -> typename Container::value_type* { return c.data(); } #if defined(_SECURE_SCL) && _SECURE_SCL // Make a checked iterator to avoid MSVC warnings. template using checked_ptr = stdext::checked_array_iterator; -template checked_ptr make_checked(T* p, size_t size) { +template auto make_checked(T* p, size_t size) -> checked_ptr { return {p, size}; } #else template using checked_ptr = T*; -template inline T* make_checked(T* p, size_t) { return p; } +template inline auto make_checked(T* p, size_t) -> T* { return p; } #endif +// Attempts to reserve space for n extra characters in the output range. +// Returns a pointer to the reserved range or a reference to it. template ::value)> -#if FMT_CLANG_VERSION +#if FMT_CLANG_VERSION >= 307 && !FMT_ICC_VERSION __attribute__((no_sanitize("undefined"))) #endif -inline checked_ptr -reserve(std::back_insert_iterator it, size_t n) { +inline auto +reserve(std::back_insert_iterator it, size_t n) + -> checked_ptr { Container& c = get_container(it); size_t size = c.size(); c.resize(size + n); @@ -378,21 +363,26 @@ } template -inline buffer_appender reserve(buffer_appender it, size_t n) { +inline auto reserve(buffer_appender it, size_t n) -> buffer_appender { buffer& buf = get_container(it); buf.try_reserve(buf.size() + n); return it; } -template inline Iterator& reserve(Iterator& it, size_t) { +template +constexpr auto reserve(Iterator& it, size_t) -> Iterator& { return it; } +template +using reserve_iterator = + remove_reference_t(), 0))>; + template -constexpr T* to_pointer(OutputIt, size_t) { +constexpr auto to_pointer(OutputIt, size_t) -> T* { return nullptr; } -template T* to_pointer(buffer_appender it, size_t n) { +template auto to_pointer(buffer_appender it, size_t n) -> T* { buffer& buf = get_container(it); auto size = buf.size(); if (buf.capacity() < size + n) return nullptr; @@ -401,192 +391,179 @@ } template ::value)> -inline std::back_insert_iterator base_iterator( - std::back_insert_iterator& it, - checked_ptr) { +inline auto base_iterator(std::back_insert_iterator& it, + checked_ptr) + -> std::back_insert_iterator { return it; } template -inline Iterator base_iterator(Iterator, Iterator it) { +constexpr auto base_iterator(Iterator, Iterator it) -> Iterator { return it; } -// An output iterator that counts the number of objects written to it and -// discards them. -class counting_iterator { - private: - size_t count_; - - public: - using iterator_category = std::output_iterator_tag; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = void; - using _Unchecked_type = counting_iterator; // Mark iterator as checked. - - struct value_type { - template void operator=(const T&) {} - }; - - counting_iterator() : count_(0) {} - - size_t count() const { return count_; } - - counting_iterator& operator++() { - ++count_; - return *this; - } - counting_iterator operator++(int) { - auto it = *this; - ++*this; - return it; - } - - friend counting_iterator operator+(counting_iterator it, difference_type n) { - it.count_ += static_cast(n); - return it; - } - - value_type operator*() const { return {}; } -}; - -template class truncating_iterator_base { - protected: - OutputIt out_; - size_t limit_; - size_t count_; - - truncating_iterator_base(OutputIt out, size_t limit) - : out_(out), limit_(limit), count_(0) {} - - public: - using iterator_category = std::output_iterator_tag; - using value_type = typename std::iterator_traits::value_type; - using difference_type = void; - using pointer = void; - using reference = void; - using _Unchecked_type = - truncating_iterator_base; // Mark iterator as checked. - - OutputIt base() const { return out_; } - size_t count() const { return count_; } -}; - -// An output iterator that truncates the output and counts the number of objects -// written to it. -template ::value_type>::type> -class truncating_iterator; - -template -class truncating_iterator - : public truncating_iterator_base { - mutable typename truncating_iterator_base::value_type blackhole_; - - public: - using value_type = typename truncating_iterator_base::value_type; - - truncating_iterator(OutputIt out, size_t limit) - : truncating_iterator_base(out, limit) {} - - truncating_iterator& operator++() { - if (this->count_++ < this->limit_) ++this->out_; - return *this; - } - - truncating_iterator operator++(int) { - auto it = *this; - ++*this; - return it; - } - - value_type& operator*() const { - return this->count_ < this->limit_ ? *this->out_ : blackhole_; +// is spectacularly slow to compile in C++20 so use a simple fill_n +// instead (#1998). +template +FMT_CONSTEXPR auto fill_n(OutputIt out, Size count, const T& value) + -> OutputIt { + for (Size i = 0; i < count; ++i) *out++ = value; + return out; +} +template +FMT_CONSTEXPR20 auto fill_n(T* out, Size count, char value) -> T* { + if (is_constant_evaluated()) { + return fill_n(out, count, value); } -}; + std::memset(out, value, to_unsigned(count)); + return out + count; +} -template -class truncating_iterator - : public truncating_iterator_base { - public: - truncating_iterator(OutputIt out, size_t limit) - : truncating_iterator_base(out, limit) {} +#ifdef __cpp_char8_t +using char8_type = char8_t; +#else +enum char8_type : unsigned char {}; +#endif - template truncating_iterator& operator=(T val) { - if (this->count_++ < this->limit_) *this->out_++ = val; - return *this; +template +FMT_CONSTEXPR FMT_NOINLINE auto copy_str_noinline(InputIt begin, InputIt end, + OutputIt out) -> OutputIt { + return copy_str(begin, end, out); +} + +// A public domain branchless UTF-8 decoder by Christopher Wellons: +// https://github.com/skeeto/branchless-utf8 +/* Decode the next character, c, from s, reporting errors in e. + * + * Since this is a branchless decoder, four bytes will be read from the + * buffer regardless of the actual length of the next character. This + * means the buffer _must_ have at least three bytes of zero padding + * following the end of the data stream. + * + * Errors are reported in e, which will be non-zero if the parsed + * character was somehow invalid: invalid byte sequence, non-canonical + * encoding, or a surrogate half. + * + * The function returns a pointer to the next character. When an error + * occurs, this pointer will be a guess that depends on the particular + * error, but it will always advance at least one byte. + */ +FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e) + -> const char* { + constexpr const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07}; + constexpr const uint32_t mins[] = {4194304, 0, 128, 2048, 65536}; + constexpr const int shiftc[] = {0, 18, 12, 6, 0}; + constexpr const int shifte[] = {0, 6, 4, 2, 0}; + + int len = code_point_length(s); + const char* next = s + len; + + // Assume a four-byte character and load four bytes. Unused bits are + // shifted out. + *c = uint32_t(s[0] & masks[len]) << 18; + *c |= uint32_t(s[1] & 0x3f) << 12; + *c |= uint32_t(s[2] & 0x3f) << 6; + *c |= uint32_t(s[3] & 0x3f) << 0; + *c >>= shiftc[len]; + + // Accumulate the various error conditions. + using uchar = unsigned char; + *e = (*c < mins[len]) << 6; // non-canonical encoding + *e |= ((*c >> 11) == 0x1b) << 7; // surrogate half? + *e |= (*c > 0x10FFFF) << 8; // out of range? + *e |= (uchar(s[1]) & 0xc0) >> 2; + *e |= (uchar(s[2]) & 0xc0) >> 4; + *e |= uchar(s[3]) >> 6; + *e ^= 0x2a; // top two bits of each tail byte correct? + *e >>= shifte[len]; + + return next; +} + +template +FMT_CONSTEXPR void for_each_codepoint(string_view s, F f) { + auto decode = [f](const char* p) { + auto cp = uint32_t(); + auto error = 0; + p = utf8_decode(p, &cp, &error); + f(cp, error); + return p; + }; + auto p = s.data(); + const size_t block_size = 4; // utf8_decode always reads blocks of 4 chars. + if (s.size() >= block_size) { + for (auto end = p + s.size() - block_size + 1; p < end;) p = decode(p); + } + if (auto num_chars_left = s.data() + s.size() - p) { + char buf[2 * block_size - 1] = {}; + copy_str(p, p + num_chars_left, buf); + p = buf; + do { + p = decode(p); + } while (p - buf < num_chars_left); } - - truncating_iterator& operator++() { return *this; } - truncating_iterator& operator++(int) { return *this; } - truncating_iterator& operator*() { return *this; } -}; +} template -inline size_t count_code_points(basic_string_view s) { +inline auto compute_width(basic_string_view s) -> size_t { return s.size(); } -// Counts the number of code points in a UTF-8 string. -inline size_t count_code_points(basic_string_view s) { - const char* data = s.data(); +// Computes approximate display width of a UTF-8 string. +FMT_CONSTEXPR inline size_t compute_width(string_view s) { size_t num_code_points = 0; - for (size_t i = 0, size = s.size(); i != size; ++i) { - if ((data[i] & 0xc0) != 0x80) ++num_code_points; - } + // It is not a lambda for compatibility with C++14. + struct count_code_points { + size_t* count; + FMT_CONSTEXPR void operator()(uint32_t cp, int error) const { + *count += detail::to_unsigned( + 1 + + (error == 0 && cp >= 0x1100 && + (cp <= 0x115f || // Hangul Jamo init. consonants + cp == 0x2329 || // LEFT-POINTING ANGLE BRACKET + cp == 0x232a || // RIGHT-POINTING ANGLE BRACKET + // CJK ... Yi except IDEOGRAPHIC HALF FILL SPACE: + (cp >= 0x2e80 && cp <= 0xa4cf && cp != 0x303f) || + (cp >= 0xac00 && cp <= 0xd7a3) || // Hangul Syllables + (cp >= 0xf900 && cp <= 0xfaff) || // CJK Compatibility Ideographs + (cp >= 0xfe10 && cp <= 0xfe19) || // Vertical Forms + (cp >= 0xfe30 && cp <= 0xfe6f) || // CJK Compatibility Forms + (cp >= 0xff00 && cp <= 0xff60) || // Fullwidth Forms + (cp >= 0xffe0 && cp <= 0xffe6) || // Fullwidth Forms + (cp >= 0x20000 && cp <= 0x2fffd) || // CJK + (cp >= 0x30000 && cp <= 0x3fffd) || + // Miscellaneous Symbols and Pictographs + Emoticons: + (cp >= 0x1f300 && cp <= 0x1f64f) || + // Supplemental Symbols and Pictographs: + (cp >= 0x1f900 && cp <= 0x1f9ff)))); + } + }; + for_each_codepoint(s, count_code_points{&num_code_points}); return num_code_points; } -inline size_t count_code_points(basic_string_view s) { - return count_code_points(basic_string_view( +inline auto compute_width(basic_string_view s) -> size_t { + return compute_width(basic_string_view( reinterpret_cast(s.data()), s.size())); } template -inline size_t code_point_index(basic_string_view s, size_t n) { +inline auto code_point_index(basic_string_view s, size_t n) -> size_t { size_t size = s.size(); return n < size ? n : size; } // Calculates the index of the nth code point in a UTF-8 string. -inline size_t code_point_index(basic_string_view s, size_t n) { +inline auto code_point_index(basic_string_view s, size_t n) + -> size_t { const char8_type* data = s.data(); size_t num_code_points = 0; for (size_t i = 0, size = s.size(); i != size; ++i) { - if ((data[i] & 0xc0) != 0x80 && ++num_code_points > n) { - return i; - } + if ((data[i] & 0xc0) != 0x80 && ++num_code_points > n) return i; } return s.size(); } -template -using needs_conversion = bool_constant< - std::is_same::value_type, - char>::value && - std::is_same::value>; - -template ::value)> -OutputIt copy_str(InputIt begin, InputIt end, OutputIt it) { - return std::copy(begin, end, it); -} - -template ::value)> -OutputIt copy_str(InputIt begin, InputIt end, OutputIt it) { - return std::transform(begin, end, it, - [](char c) { return static_cast(c); }); -} - -template -inline counting_iterator copy_str(InputIt begin, InputIt end, - counting_iterator it) { - return it + (end - begin); -} - template using is_fast_float = bool_constant::is_iec559 && sizeof(T) <= sizeof(double)>; @@ -598,7 +575,7 @@ template template void buffer::append(const U* begin, const U* end) { - do { + while (begin != end) { auto count = to_unsigned(end - begin); try_reserve(size_ + count); auto free_cap = capacity_ - size_; @@ -606,16 +583,17 @@ std::uninitialized_copy_n(begin, count, make_checked(ptr_ + size_, count)); size_ += count; begin += count; - } while (begin != end); + } } -template -void iterator_buffer::flush() { - out_ = std::copy_n(data_, this->limit(this->size()), out_); - this->clear(); -} +template +struct is_locale : std::false_type {}; +template +struct is_locale> : std::true_type {}; } // namespace detail +FMT_MODULE_EXPORT_BEGIN + // The number of characters to store in the basic_memory_buffer object itself // to avoid dynamic memory allocation. enum { inline_buffer_size = 500 }; @@ -625,15 +603,7 @@ A dynamically growing memory buffer for trivially copyable/constructible types with the first ``SIZE`` elements stored in the object itself. - You can use one of the following type aliases for common character types: - - +----------------+------------------------------+ - | Type | Definition | - +================+==============================+ - | memory_buffer | basic_memory_buffer | - +----------------+------------------------------+ - | wmemory_buffer | basic_memory_buffer | - +----------------+------------------------------+ + You can use the ``memory_buffer`` type alias for ``char`` instead. **Example**:: @@ -710,7 +680,8 @@ Moves the content of the other ``basic_memory_buffer`` object to this one. \endrst */ - basic_memory_buffer& operator=(basic_memory_buffer&& other) FMT_NOEXCEPT { + auto operator=(basic_memory_buffer&& other) FMT_NOEXCEPT + -> basic_memory_buffer& { FMT_ASSERT(this != &other, ""); deallocate(); move(other); @@ -718,7 +689,7 @@ } // Returns a copy of the allocator associated with this buffer. - Allocator get_allocator() const { return alloc_; } + auto get_allocator() const -> Allocator { return alloc_; } /** Resizes the buffer to contain *count* elements. If T is a POD type new @@ -742,9 +713,13 @@ #ifdef FMT_FUZZ if (size > 5000) throw std::runtime_error("fuzz mode - won't grow that much"); #endif + const size_t max_size = std::allocator_traits::max_size(alloc_); size_t old_capacity = this->capacity(); size_t new_capacity = old_capacity + old_capacity / 2; - if (size > new_capacity) new_capacity = size; + if (size > new_capacity) + new_capacity = size; + else if (new_capacity > max_size) + new_capacity = size > max_size ? size : max_size; T* old_data = this->data(); T* new_data = std::allocator_traits::allocate(alloc_, new_capacity); @@ -759,12 +734,15 @@ } using memory_buffer = basic_memory_buffer; -using wmemory_buffer = basic_memory_buffer; template struct is_contiguous> : std::true_type { }; +namespace detail { +FMT_API void print(std::FILE*, string_view); +} + /** A formatting error such as invalid format string. */ FMT_CLASS_API class FMT_API format_error : public std::runtime_error { @@ -776,10 +754,66 @@ format_error& operator=(const format_error&) = default; format_error(format_error&&) = default; format_error& operator=(format_error&&) = default; - ~format_error() FMT_NOEXCEPT FMT_OVERRIDE; + ~format_error() FMT_NOEXCEPT FMT_OVERRIDE FMT_MSC_DEFAULT; }; -namespace detail { +/** + \rst + Constructs a `~fmt::format_arg_store` object that contains references + to arguments and can be implicitly converted to `~fmt::format_args`. + If ``fmt`` is a compile-time string then `make_args_checked` checks + its validity at compile time. + \endrst + */ +template > +FMT_INLINE auto make_args_checked(const S& fmt, + const remove_reference_t&... args) + -> format_arg_store, remove_reference_t...> { + static_assert( + detail::count<( + std::is_base_of>::value && + std::is_reference::value)...>() == 0, + "passing views as lvalues is disallowed"); + detail::check_format_string(fmt); + return {args...}; +} + +// compile-time support +namespace detail_exported { +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS +template struct fixed_string { + constexpr fixed_string(const Char (&str)[N]) { + detail::copy_str(static_cast(str), + str + N, data); + } + Char data[N]{}; +}; +#endif + +// Converts a compile-time string to basic_string_view. +template +constexpr auto compile_string_to_view(const Char (&s)[N]) + -> basic_string_view { + // Remove trailing NUL character if needed. Won't be present if this is used + // with a raw character array (i.e. not defined as a string). + return {s, N - (std::char_traits::to_int_type(s[N - 1]) == 0 ? 1 : 0)}; +} +template +constexpr auto compile_string_to_view(detail::std_string_view s) + -> basic_string_view { + return {s.data(), s.size()}; +} +} // namespace detail_exported + +FMT_BEGIN_DETAIL_NAMESPACE + +inline void throw_format_error(const char* message) { + FMT_THROW(format_error(message)); +} + +template struct is_integral : std::is_integral {}; +template <> struct is_integral : std::true_type {}; +template <> struct is_integral : std::true_type {}; template using is_signed = @@ -789,16 +823,16 @@ // Returns true if value is negative, false otherwise. // Same as `value < 0` but doesn't produce warnings if T is an unsigned type. template ::value)> -FMT_CONSTEXPR bool is_negative(T value) { +FMT_CONSTEXPR auto is_negative(T value) -> bool { return value < 0; } template ::value)> -FMT_CONSTEXPR bool is_negative(T) { +FMT_CONSTEXPR auto is_negative(T) -> bool { return false; } template ::value)> -FMT_CONSTEXPR bool is_supported_floating_point(T) { +FMT_CONSTEXPR auto is_supported_floating_point(T) -> uint16_t { return (std::is_same::value && FMT_USE_FLOAT) || (std::is_same::value && FMT_USE_DOUBLE) || (std::is_same::value && FMT_USE_LONG_DOUBLE); @@ -811,121 +845,58 @@ conditional_t() <= 32 && !FMT_REDUCE_INT_INSTANTIATIONS, uint32_t, conditional_t() <= 64, uint64_t, uint128_t>>; +template +using uint64_or_128_t = conditional_t() <= 64, uint64_t, uint128_t>; -// 128-bit integer type used internally -struct FMT_EXTERN_TEMPLATE_API uint128_wrapper { - uint128_wrapper() = default; - -#if FMT_USE_INT128 - uint128_t internal_; - - uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT - : internal_{static_cast(low) | - (static_cast(high) << 64)} {} - - uint128_wrapper(uint128_t u) : internal_{u} {} - - uint64_t high() const FMT_NOEXCEPT { return uint64_t(internal_ >> 64); } - uint64_t low() const FMT_NOEXCEPT { return uint64_t(internal_); } - - uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { - internal_ += n; - return *this; - } -#else - uint64_t high_; - uint64_t low_; - - uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT : high_{high}, - low_{low} {} - - uint64_t high() const FMT_NOEXCEPT { return high_; } - uint64_t low() const FMT_NOEXCEPT { return low_; } - - uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { -# if defined(_MSC_VER) && defined(_M_X64) - unsigned char carry = _addcarry_u64(0, low_, n, &low_); - _addcarry_u64(carry, high_, 0, &high_); - return *this; -# else - uint64_t sum = low_ + n; - high_ += (sum < low_ ? 1 : 0); - low_ = sum; - return *this; -# endif - } -#endif -}; - -// Table entry type for divisibility test used internally -template struct FMT_EXTERN_TEMPLATE_API divtest_table_entry { - T mod_inv; - T max_quotient; -}; +#define FMT_POWERS_OF_10(factor) \ + factor * 10, (factor)*100, (factor)*1000, (factor)*10000, (factor)*100000, \ + (factor)*1000000, (factor)*10000000, (factor)*100000000, \ + (factor)*1000000000 // Static data is placed in this class template for the header-only config. -template struct FMT_EXTERN_TEMPLATE_API basic_data { - static const uint64_t powers_of_10_64[]; - static const uint32_t zero_or_powers_of_10_32_new[]; - static const uint64_t zero_or_powers_of_10_64_new[]; - static const uint64_t grisu_pow10_significands[]; - static const int16_t grisu_pow10_exponents[]; - static const divtest_table_entry divtest_table_for_pow5_32[]; - static const divtest_table_entry divtest_table_for_pow5_64[]; - static const uint64_t dragonbox_pow10_significands_64[]; - static const uint128_wrapper dragonbox_pow10_significands_128[]; +template struct basic_data { // log10(2) = 0x0.4d104d427de7fbcc... static const uint64_t log10_2_significand = 0x4d104d427de7fbcc; -#if !FMT_USE_FULL_CACHE_DRAGONBOX - static const uint64_t powers_of_5_64[]; - static const uint32_t dragonbox_pow10_recovery_errors[]; -#endif - // GCC generates slightly better code for pairs than chars. - using digit_pair = char[2]; - static const digit_pair digits[]; - static const char hex_digits[]; - static const char foreground_color[]; - static const char background_color[]; - static const char reset_color[5]; - static const wchar_t wreset_color[5]; - static const char signs[]; - static const char left_padding_shifts[5]; - static const char right_padding_shifts[5]; - - // DEPRECATED! These are for ABI compatibility. - static const uint32_t zero_or_powers_of_10_32[]; - static const uint64_t zero_or_powers_of_10_64[]; -}; - -// Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)). -// This is a function instead of an array to workaround a bug in GCC10 (#1810). -FMT_INLINE uint16_t bsr2log10(int bsr) { - static constexpr uint16_t data[] = { - 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, - 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, - 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15, - 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20}; - return data[bsr]; -} -#ifndef FMT_EXPORTED -FMT_EXTERN template struct basic_data; + // GCC generates slightly better code for pairs than chars. + FMT_API static constexpr const char digits[100][2] = { + {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'}, + {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, {'1', '0'}, {'1', '1'}, + {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'}, + {'1', '8'}, {'1', '9'}, {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, + {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'}, + {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'}, + {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, {'4', '0'}, {'4', '1'}, + {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'}, + {'4', '8'}, {'4', '9'}, {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, + {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'}, + {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'}, + {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, {'7', '0'}, {'7', '1'}, + {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'}, + {'7', '8'}, {'7', '9'}, {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, + {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'}, + {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'}, + {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}}; + + FMT_API static constexpr const char hex_digits[] = "0123456789abcdef"; + FMT_API static constexpr const char signs[4] = {0, '-', '+', ' '}; + FMT_API static constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+', + 0x1000000u | ' '}; + FMT_API static constexpr const char left_padding_shifts[5] = {31, 31, 0, 1, + 0}; + FMT_API static constexpr const char right_padding_shifts[5] = {0, 31, 0, 1, + 0}; +}; + +#ifdef FMT_SHARED +// Required for -flto, -fivisibility=hidden and -shared to work +extern template struct basic_data; #endif // This is a struct rather than an alias to avoid shadowing warnings in gcc. struct data : basic_data<> {}; -#ifdef FMT_BUILTIN_CLZLL -// Returns the number of decimal digits in n. Leading zeros are not counted -// except for n == 0 in which case count_digits returns 1. -inline int count_digits(uint64_t n) { - // https://github.com/fmtlib/format-benchmark/blob/master/digits10 - auto t = bsr2log10(FMT_BUILTIN_CLZLL(n | 1) ^ 63); - return t - (n < data::zero_or_powers_of_10_64_new[t]); -} -#else -// Fallback version of count_digits used when __builtin_clz is not available. -inline int count_digits(uint64_t n) { +template FMT_CONSTEXPR auto count_digits_fallback(T n) -> int { int count = 1; for (;;) { // Integer division is slow so do it for a group of four digits instead @@ -939,27 +910,41 @@ count += 4; } } +#if FMT_USE_INT128 +FMT_CONSTEXPR inline auto count_digits(uint128_t n) -> int { + return count_digits_fallback(n); +} #endif -#if FMT_USE_INT128 -inline int count_digits(uint128_t n) { - int count = 1; - for (;;) { - // Integer division is slow so do it for a group of four digits instead - // of for every digit. The idea comes from the talk by Alexandrescu - // "Three Optimization Tips for C++". See speed-test for a comparison. - if (n < 10) return count; - if (n < 100) return count + 1; - if (n < 1000) return count + 2; - if (n < 10000) return count + 3; - n /= 10000U; - count += 4; +// Returns the number of decimal digits in n. Leading zeros are not counted +// except for n == 0 in which case count_digits returns 1. +FMT_CONSTEXPR20 inline auto count_digits(uint64_t n) -> int { +#ifdef FMT_BUILTIN_CLZLL + if (!is_constant_evaluated()) { + // https://github.com/fmtlib/format-benchmark/blob/master/digits10 + // Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)). + constexpr uint16_t bsr2log10[] = { + 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, + 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, + 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15, + 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20}; + auto t = bsr2log10[FMT_BUILTIN_CLZLL(n | 1) ^ 63]; + constexpr const uint64_t zero_or_powers_of_10[] = { + 0, 0, FMT_POWERS_OF_10(1U), FMT_POWERS_OF_10(1000000000ULL), + 10000000000000000000ULL}; + return t - (n < zero_or_powers_of_10[t]); } -} #endif + return count_digits_fallback(n); +} // Counts the number of digits in n. BITS = log2(radix). -template inline int count_digits(UInt n) { +template +FMT_CONSTEXPR auto count_digits(UInt n) -> int { +#ifdef FMT_BUILTIN_CLZ + if (num_bits() == 32) + return (FMT_BUILTIN_CLZ(static_cast(n) | 1) ^ 31) / BITS + 1; +#endif int num_digits = 0; do { ++num_digits; @@ -967,66 +952,82 @@ return num_digits; } -template <> int count_digits<4>(detail::fallback_uintptr n); +template <> auto count_digits<4>(detail::fallback_uintptr n) -> int; -#if FMT_GCC_VERSION || FMT_CLANG_VERSION -# define FMT_ALWAYS_INLINE inline __attribute__((always_inline)) -#elif FMT_MSC_VER -# define FMT_ALWAYS_INLINE __forceinline -#else -# define FMT_ALWAYS_INLINE inline -#endif - -// To suppress unnecessary security cookie checks -#if FMT_MSC_VER && !FMT_CLANG_VERSION -# define FMT_SAFEBUFFERS __declspec(safebuffers) -#else -# define FMT_SAFEBUFFERS -#endif +// It is a separate function rather than a part of count_digits to workaround +// the lack of static constexpr in constexpr functions. +FMT_INLINE uint64_t count_digits_inc(int n) { + // An optimization by Kendall Willets from https://bit.ly/3uOIQrB. + // This increments the upper 32 bits (log10(T) - 1) when >= T is added. +#define FMT_INC(T) (((sizeof(#T) - 1ull) << 32) - T) + static constexpr uint64_t table[] = { + FMT_INC(0), FMT_INC(0), FMT_INC(0), // 8 + FMT_INC(10), FMT_INC(10), FMT_INC(10), // 64 + FMT_INC(100), FMT_INC(100), FMT_INC(100), // 512 + FMT_INC(1000), FMT_INC(1000), FMT_INC(1000), // 4096 + FMT_INC(10000), FMT_INC(10000), FMT_INC(10000), // 32k + FMT_INC(100000), FMT_INC(100000), FMT_INC(100000), // 256k + FMT_INC(1000000), FMT_INC(1000000), FMT_INC(1000000), // 2048k + FMT_INC(10000000), FMT_INC(10000000), FMT_INC(10000000), // 16M + FMT_INC(100000000), FMT_INC(100000000), FMT_INC(100000000), // 128M + FMT_INC(1000000000), FMT_INC(1000000000), FMT_INC(1000000000), // 1024M + FMT_INC(1000000000), FMT_INC(1000000000) // 4B + }; + return table[n]; +} -#ifdef FMT_BUILTIN_CLZ // Optional version of count_digits for better performance on 32-bit platforms. -inline int count_digits(uint32_t n) { - auto t = bsr2log10(FMT_BUILTIN_CLZ(n | 1) ^ 31); - return t - (n < data::zero_or_powers_of_10_32_new[t]); -} +FMT_CONSTEXPR20 inline auto count_digits(uint32_t n) -> int { +#ifdef FMT_BUILTIN_CLZ + if (!is_constant_evaluated()) { + auto inc = count_digits_inc(FMT_BUILTIN_CLZ(n | 1) ^ 31); + return static_cast((n + inc) >> 32); + } #endif + return count_digits_fallback(n); +} -template constexpr int digits10() FMT_NOEXCEPT { +template constexpr auto digits10() FMT_NOEXCEPT -> int { return std::numeric_limits::digits10; } -template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } -template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } - -template FMT_API std::string grouping_impl(locale_ref loc); -template inline std::string grouping(locale_ref loc) { - return grouping_impl(loc); +template <> constexpr auto digits10() FMT_NOEXCEPT -> int { + return 38; } -template <> inline std::string grouping(locale_ref loc) { - return grouping_impl(loc); +template <> constexpr auto digits10() FMT_NOEXCEPT -> int { + return 38; } -template FMT_API Char thousands_sep_impl(locale_ref loc); -template inline Char thousands_sep(locale_ref loc) { - return Char(thousands_sep_impl(loc)); +template struct thousands_sep_result { + std::string grouping; + Char thousands_sep; +}; + +template +FMT_API auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result; +template +inline auto thousands_sep(locale_ref loc) -> thousands_sep_result { + auto result = thousands_sep_impl(loc); + return {result.grouping, Char(result.thousands_sep)}; } -template <> inline wchar_t thousands_sep(locale_ref loc) { +template <> +inline auto thousands_sep(locale_ref loc) -> thousands_sep_result { return thousands_sep_impl(loc); } -template FMT_API Char decimal_point_impl(locale_ref loc); -template inline Char decimal_point(locale_ref loc) { +template +FMT_API auto decimal_point_impl(locale_ref loc) -> Char; +template inline auto decimal_point(locale_ref loc) -> Char { return Char(decimal_point_impl(loc)); } -template <> inline wchar_t decimal_point(locale_ref loc) { +template <> inline auto decimal_point(locale_ref loc) -> wchar_t { return decimal_point_impl(loc); } // Compares two characters for equality. -template bool equal2(const Char* lhs, const char* rhs) { - return lhs[0] == rhs[0] && lhs[1] == rhs[1]; +template auto equal2(const Char* lhs, const char* rhs) -> bool { + return lhs[0] == Char(rhs[0]) && lhs[1] == Char(rhs[1]); } -inline bool equal2(const char* lhs, const char* rhs) { +inline auto equal2(const char* lhs, const char* rhs) -> bool { return memcmp(lhs, rhs, 2) == 0; } @@ -1046,11 +1047,19 @@ // buffer of specified size. The caller must ensure that the buffer is large // enough. template -inline format_decimal_result format_decimal(Char* out, UInt value, - int size) { +FMT_CONSTEXPR20 auto format_decimal(Char* out, UInt value, int size) + -> format_decimal_result { FMT_ASSERT(size >= count_digits(value), "invalid digit count"); out += size; Char* end = out; + if (is_constant_evaluated()) { + while (value >= 10) { + *--out = static_cast('0' + value % 10); + value /= 10; + } + *--out = static_cast('0' + value); + return {out, end}; + } while (value >= 100) { // Integer division is slow so do it for a group of two digits instead // of for every digit. The idea comes from the talk by Alexandrescu @@ -1070,17 +1079,17 @@ template >::value)> -inline format_decimal_result format_decimal(Iterator out, UInt value, - int size) { +inline auto format_decimal(Iterator out, UInt value, int size) + -> format_decimal_result { // Buffer is large enough to hold all digits (digits10 + 1). Char buffer[digits10() + 1]; auto end = format_decimal(buffer, value, size).end; - return {out, detail::copy_str(buffer, end, out)}; + return {out, detail::copy_str_noinline(buffer, end, out)}; } template -inline Char* format_uint(Char* buffer, UInt value, int num_digits, - bool upper = false) { +FMT_CONSTEXPR auto format_uint(Char* buffer, UInt value, int num_digits, + bool upper = false) -> Char* { buffer += num_digits; Char* end = buffer; do { @@ -1093,8 +1102,8 @@ } template -Char* format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, - bool = false) { +auto format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, + bool = false) -> Char* { auto char_digits = std::numeric_limits::digits / 4; int start = (num_digits + char_digits - 1) / char_digits - 1; if (int start_digits = num_digits % char_digits) { @@ -1115,7 +1124,8 @@ } template -inline It format_uint(It out, UInt value, int num_digits, bool upper = false) { +inline auto format_uint(It out, UInt value, int num_digits, bool upper = false) + -> It { if (auto ptr = to_pointer(out, to_unsigned(num_digits))) { format_uint(ptr, value, num_digits, upper); return out; @@ -1123,86 +1133,22 @@ // Buffer should be large enough to hold all digits (digits / BASE_BITS + 1). char buffer[num_bits() / BASE_BITS + 1]; format_uint(buffer, value, num_digits, upper); - return detail::copy_str(buffer, buffer + num_digits, out); + return detail::copy_str_noinline(buffer, buffer + num_digits, out); } // A converter from UTF-8 to UTF-16. class utf8_to_utf16 { private: - wmemory_buffer buffer_; + basic_memory_buffer buffer_; public: FMT_API explicit utf8_to_utf16(string_view s); - operator wstring_view() const { return {&buffer_[0], size()}; } - size_t size() const { return buffer_.size() - 1; } - const wchar_t* c_str() const { return &buffer_[0]; } - std::wstring str() const { return {&buffer_[0], size()}; } -}; - -template struct null {}; - -// Workaround an array initialization issue in gcc 4.8. -template struct fill_t { - private: - enum { max_size = 4 }; - Char data_[max_size] = {Char(' '), Char(0), Char(0), Char(0)}; - unsigned char size_ = 1; - - public: - FMT_CONSTEXPR void operator=(basic_string_view s) { - auto size = s.size(); - if (size > max_size) { - FMT_THROW(format_error("invalid fill")); - return; - } - for (size_t i = 0; i < size; ++i) data_[i] = s[i]; - size_ = static_cast(size); - } - - size_t size() const { return size_; } - const Char* data() const { return data_; } - - FMT_CONSTEXPR Char& operator[](size_t index) { return data_[index]; } - FMT_CONSTEXPR const Char& operator[](size_t index) const { - return data_[index]; - } -}; -} // namespace detail - -// We cannot use enum classes as bit fields because of a gcc bug -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414. -namespace align { -enum type { none, left, right, center, numeric }; -} -using align_t = align::type; - -namespace sign { -enum type { none, minus, plus, space }; -} -using sign_t = sign::type; - -// Format specifiers for built-in and string types. -template struct basic_format_specs { - int width; - int precision; - char type; - align_t align : 4; - sign_t sign : 3; - bool alt : 1; // Alternate form ('#'). - detail::fill_t fill; - - constexpr basic_format_specs() - : width(0), - precision(-1), - type(0), - align(align::none), - sign(sign::none), - alt(false) {} + operator basic_string_view() const { return {&buffer_[0], size()}; } + auto size() const -> size_t { return buffer_.size() - 1; } + auto c_str() const -> const wchar_t* { return &buffer_[0]; } + auto str() const -> std::wstring { return {&buffer_[0], size()}; } }; -using format_specs = basic_format_specs; - -namespace detail { namespace dragonbox { // Type-specific information that Dragonbox uses. @@ -1266,37 +1212,21 @@ int exponent; }; -template FMT_API decimal_fp to_decimal(T x) FMT_NOEXCEPT; +template +FMT_API auto to_decimal(T x) FMT_NOEXCEPT -> decimal_fp; } // namespace dragonbox template -constexpr typename dragonbox::float_info::carrier_uint exponent_mask() { +constexpr auto exponent_mask() -> + typename dragonbox::float_info::carrier_uint { using uint = typename dragonbox::float_info::carrier_uint; return ((uint(1) << dragonbox::float_info::exponent_bits) - 1) << dragonbox::float_info::significand_bits; } -// A floating-point presentation format. -enum class float_format : unsigned char { - general, // General: exponent notation or fixed point based on magnitude. - exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3. - fixed, // Fixed point with the default precision of 6, e.g. 0.0012. - hex -}; - -struct float_specs { - int precision; - float_format format : 8; - sign_t sign : 8; - bool upper : 1; - bool locale : 1; - bool binary32 : 1; - bool use_grisu : 1; - bool showpoint : 1; -}; - // Writes the exponent exp in the form "[+-]d{2,3}" to buffer. -template It write_exponent(int exp, It it) { +template +auto write_exponent(int exp, It it) -> It { FMT_ASSERT(-10000 < exp && exp < 10000, "exponent out of range"); if (exp < 0) { *it++ = static_cast('-'); @@ -1317,174 +1247,28 @@ } template -int format_float(T value, int precision, float_specs specs, buffer& buf); +auto format_float(T value, int precision, float_specs specs, buffer& buf) + -> int; // Formats a floating-point number with snprintf. template -int snprintf_float(T value, int precision, float_specs specs, - buffer& buf); - -template T promote_float(T value) { return value; } -inline double promote_float(float value) { return static_cast(value); } - -template -FMT_CONSTEXPR void handle_int_type_spec(char spec, Handler&& handler) { - switch (spec) { - case 0: - case 'd': - handler.on_dec(); - break; - case 'x': - case 'X': - handler.on_hex(); - break; - case 'b': - case 'B': - handler.on_bin(); - break; - case 'o': - handler.on_oct(); - break; -#ifdef FMT_DEPRECATED_N_SPECIFIER - case 'n': -#endif - case 'L': - handler.on_num(); - break; - case 'c': - handler.on_chr(); - break; - default: - handler.on_error(); - } -} - -template -FMT_CONSTEXPR float_specs parse_float_type_spec( - const basic_format_specs& specs, ErrorHandler&& eh = {}) { - auto result = float_specs(); - result.showpoint = specs.alt; - switch (specs.type) { - case 0: - result.format = float_format::general; - result.showpoint |= specs.precision > 0; - break; - case 'G': - result.upper = true; - FMT_FALLTHROUGH; - case 'g': - result.format = float_format::general; - break; - case 'E': - result.upper = true; - FMT_FALLTHROUGH; - case 'e': - result.format = float_format::exp; - result.showpoint |= specs.precision != 0; - break; - case 'F': - result.upper = true; - FMT_FALLTHROUGH; - case 'f': - result.format = float_format::fixed; - result.showpoint |= specs.precision != 0; - break; - case 'A': - result.upper = true; - FMT_FALLTHROUGH; - case 'a': - result.format = float_format::hex; - break; -#ifdef FMT_DEPRECATED_N_SPECIFIER - case 'n': -#endif - case 'L': - result.locale = true; - break; - default: - eh.on_error("invalid type specifier"); - break; - } - return result; -} - -template -FMT_CONSTEXPR void handle_char_specs(const basic_format_specs* specs, - Handler&& handler) { - if (!specs) return handler.on_char(); - if (specs->type && specs->type != 'c') return handler.on_int(); - if (specs->align == align::numeric || specs->sign != sign::none || specs->alt) - handler.on_error("invalid format specifier for char"); - handler.on_char(); -} +auto snprintf_float(T value, int precision, float_specs specs, + buffer& buf) -> int; -template -FMT_CONSTEXPR void handle_cstring_type_spec(Char spec, Handler&& handler) { - if (spec == 0 || spec == 's') - handler.on_string(); - else if (spec == 'p') - handler.on_pointer(); - else - handler.on_error("invalid type specifier"); +template auto promote_float(T value) -> T { return value; } +inline auto promote_float(float value) -> double { + return static_cast(value); } -template -FMT_CONSTEXPR void check_string_type_spec(Char spec, ErrorHandler&& eh) { - if (spec != 0 && spec != 's') eh.on_error("invalid type specifier"); -} - -template -FMT_CONSTEXPR void check_pointer_type_spec(Char spec, ErrorHandler&& eh) { - if (spec != 0 && spec != 'p') eh.on_error("invalid type specifier"); -} - -template class int_type_checker : private ErrorHandler { - public: - FMT_CONSTEXPR explicit int_type_checker(ErrorHandler eh) : ErrorHandler(eh) {} - - FMT_CONSTEXPR void on_dec() {} - FMT_CONSTEXPR void on_hex() {} - FMT_CONSTEXPR void on_bin() {} - FMT_CONSTEXPR void on_oct() {} - FMT_CONSTEXPR void on_num() {} - FMT_CONSTEXPR void on_chr() {} - - FMT_CONSTEXPR void on_error() { - ErrorHandler::on_error("invalid type specifier"); - } -}; - -template -class char_specs_checker : public ErrorHandler { - private: - char type_; - - public: - FMT_CONSTEXPR char_specs_checker(char type, ErrorHandler eh) - : ErrorHandler(eh), type_(type) {} - - FMT_CONSTEXPR void on_int() { - handle_int_type_spec(type_, int_type_checker(*this)); - } - FMT_CONSTEXPR void on_char() {} -}; - -template -class cstring_type_checker : public ErrorHandler { - public: - FMT_CONSTEXPR explicit cstring_type_checker(ErrorHandler eh) - : ErrorHandler(eh) {} - - FMT_CONSTEXPR void on_string() {} - FMT_CONSTEXPR void on_pointer() {} -}; - -template -FMT_NOINLINE OutputIt fill(OutputIt it, size_t n, const fill_t& fill) { - auto fill_size = fill.size(); - if (fill_size == 1) return std::fill_n(it, n, fill[0]); - for (size_t i = 0; i < n; ++i) it = std::copy_n(fill.data(), fill_size, it); - return it; +template +FMT_NOINLINE FMT_CONSTEXPR auto fill(OutputIt it, size_t n, + const fill_t& fill) -> OutputIt { + auto fill_size = fill.size(); + if (fill_size == 1) return detail::fill_n(it, n, fill[0]); + auto data = fill.data(); + for (size_t i = 0; i < n; ++i) + it = copy_str(data, data + fill_size, it); + return it; } // Writes the output of f, padded according to format specifications in specs. @@ -1492,39 +1276,72 @@ // width: output display width in (terminal) column positions. template -inline OutputIt write_padded(OutputIt out, - const basic_format_specs& specs, size_t size, - size_t width, F&& f) { +FMT_CONSTEXPR auto write_padded(OutputIt out, + const basic_format_specs& specs, + size_t size, size_t width, F&& f) -> OutputIt { static_assert(align == align::left || align == align::right, ""); unsigned spec_width = to_unsigned(specs.width); size_t padding = spec_width > width ? spec_width - width : 0; auto* shifts = align == align::left ? data::left_padding_shifts : data::right_padding_shifts; size_t left_padding = padding >> shifts[specs.align]; + size_t right_padding = padding - left_padding; auto it = reserve(out, size + padding * specs.fill.size()); - it = fill(it, left_padding, specs.fill); + if (left_padding != 0) it = fill(it, left_padding, specs.fill); it = f(it); - it = fill(it, padding - left_padding, specs.fill); + if (right_padding != 0) it = fill(it, right_padding, specs.fill); return base_iterator(out, it); } template -inline OutputIt write_padded(OutputIt out, - const basic_format_specs& specs, size_t size, - F&& f) { +constexpr auto write_padded(OutputIt out, const basic_format_specs& specs, + size_t size, F&& f) -> OutputIt { return write_padded(out, specs, size, size, f); } +template +FMT_CONSTEXPR auto write_bytes(OutputIt out, string_view bytes, + const basic_format_specs& specs) + -> OutputIt { + return write_padded( + out, specs, bytes.size(), [bytes](reserve_iterator it) { + const char* data = bytes.data(); + return copy_str(data, data + bytes.size(), it); + }); +} + +template +auto write_ptr(OutputIt out, UIntPtr value, + const basic_format_specs* specs) -> OutputIt { + int num_digits = count_digits<4>(value); + auto size = to_unsigned(num_digits) + size_t(2); + auto write = [=](reserve_iterator it) { + *it++ = static_cast('0'); + *it++ = static_cast('x'); + return format_uint<4, Char>(it, value, num_digits); + }; + return specs ? write_padded(out, *specs, size, write) + : base_iterator(out, write(reserve(out, size))); +} + template -OutputIt write_bytes(OutputIt out, string_view bytes, - const basic_format_specs& specs) { - using iterator = remove_reference_t; - return write_padded(out, specs, bytes.size(), [bytes](iterator it) { - const char* data = bytes.data(); - return copy_str(data, data + bytes.size(), it); +FMT_CONSTEXPR auto write_char(OutputIt out, Char value, + const basic_format_specs& specs) + -> OutputIt { + return write_padded(out, specs, 1, [=](reserve_iterator it) { + *it++ = value; + return it; }); } +template +FMT_CONSTEXPR auto write(OutputIt out, Char value, + const basic_format_specs& specs, + locale_ref loc = {}) -> OutputIt { + return check_char_specs(specs) + ? write_char(out, value, specs) + : write(out, static_cast(value), specs, loc); +} // Data for write_int that doesn't depend on output iterator type. It is used to // avoid template code bloat. @@ -1532,9 +1349,9 @@ size_t size; size_t padding; - write_int_data(int num_digits, string_view prefix, - const basic_format_specs& specs) - : size(prefix.size() + to_unsigned(num_digits)), padding(0) { + FMT_CONSTEXPR write_int_data(int num_digits, unsigned prefix, + const basic_format_specs& specs) + : size((prefix >> 24) + to_unsigned(num_digits)), padding(0) { if (specs.align == align::numeric) { auto width = to_unsigned(specs.width); if (width > size) { @@ -1542,7 +1359,7 @@ size = width; } } else if (specs.precision > num_digits) { - size = prefix.size() + to_unsigned(specs.precision); + size = (prefix >> 24) + to_unsigned(specs.precision); padding = to_unsigned(specs.precision - num_digits); } } @@ -1550,182 +1367,233 @@ // Writes an integer in the format // -// where are written by f(it). -template -OutputIt write_int(OutputIt out, int num_digits, string_view prefix, - const basic_format_specs& specs, F f) { +// where are written by write_digits(it). +// prefix contains chars in three lower bytes and the size in the fourth byte. +template +FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits, + unsigned prefix, + const basic_format_specs& specs, + W write_digits) -> OutputIt { + // Slightly faster check for specs.width == 0 && specs.precision == -1. + if ((specs.width | (specs.precision + 1)) == 0) { + auto it = reserve(out, to_unsigned(num_digits) + (prefix >> 24)); + if (prefix != 0) { + for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8) + *it++ = static_cast(p & 0xff); + } + return base_iterator(out, write_digits(it)); + } auto data = write_int_data(num_digits, prefix, specs); - using iterator = remove_reference_t; - return write_padded(out, specs, data.size, [=](iterator it) { - if (prefix.size() != 0) - it = copy_str(prefix.begin(), prefix.end(), it); - it = std::fill_n(it, data.padding, static_cast('0')); - return f(it); - }); -} - -template -OutputIt write(OutputIt out, basic_string_view s, - const basic_format_specs& specs) { - auto data = s.data(); - auto size = s.size(); - if (specs.precision >= 0 && to_unsigned(specs.precision) < size) - size = code_point_index(s, to_unsigned(specs.precision)); - auto width = specs.width != 0 - ? count_code_points(basic_string_view(data, size)) - : 0; - using iterator = remove_reference_t; - return write_padded(out, specs, size, width, [=](iterator it) { - return copy_str(data, data + size, it); - }); + return write_padded( + out, specs, data.size, [=](reserve_iterator it) { + for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8) + *it++ = static_cast(p & 0xff); + it = detail::fill_n(it, data.padding, static_cast('0')); + return write_digits(it); + }); +} + +template +auto write_int_localized(OutputIt& out, UInt value, unsigned prefix, + const basic_format_specs& specs, locale_ref loc) + -> bool { + static_assert(std::is_same, UInt>::value, ""); + const auto sep_size = 1; + auto ts = thousands_sep(loc); + if (!ts.thousands_sep) return false; + int num_digits = count_digits(value); + int size = num_digits, n = num_digits; + const std::string& groups = ts.grouping; + std::string::const_iterator group = groups.cbegin(); + while (group != groups.cend() && n > *group && *group > 0 && + *group != max_value()) { + size += sep_size; + n -= *group; + ++group; + } + if (group == groups.cend()) size += sep_size * ((n - 1) / groups.back()); + char digits[40]; + format_decimal(digits, value, num_digits); + basic_memory_buffer buffer; + if (prefix != 0) ++size; + const auto usize = to_unsigned(size); + buffer.resize(usize); + basic_string_view s(&ts.thousands_sep, sep_size); + // Index of a decimal digit with the least significant digit having index 0. + int digit_index = 0; + group = groups.cbegin(); + auto p = buffer.data() + size - 1; + for (int i = num_digits - 1; i > 0; --i) { + *p-- = static_cast(digits[i]); + if (*group <= 0 || ++digit_index % *group != 0 || + *group == max_value()) + continue; + if (group + 1 != groups.cend()) { + digit_index = 0; + ++group; + } + std::uninitialized_copy(s.data(), s.data() + s.size(), + make_checked(p, s.size())); + p -= s.size(); + } + *p-- = static_cast(*digits); + if (prefix != 0) *p = static_cast(prefix); + auto data = buffer.data(); + out = write_padded( + out, specs, usize, usize, [=](reserve_iterator it) { + return copy_str(data, data + size, it); + }); + return true; +} + +FMT_CONSTEXPR inline void prefix_append(unsigned& prefix, unsigned value) { + prefix |= prefix != 0 ? value << 8 : value; + prefix += (1u + (value > 0xff ? 1 : 0)) << 24; } -// The handle_int_type_spec handler that writes an integer. -template struct int_writer { - OutputIt out; - locale_ref locale; - const basic_format_specs& specs; +template struct write_int_arg { UInt abs_value; - char prefix[4]; - unsigned prefix_size; - - using iterator = - remove_reference_t(), 0))>; - - string_view get_prefix() const { return string_view(prefix, prefix_size); } + unsigned prefix; +}; - template - int_writer(OutputIt output, locale_ref loc, Int value, - const basic_format_specs& s) - : out(output), - locale(loc), - specs(s), - abs_value(static_cast(value)), - prefix_size(0) { - static_assert(std::is_same, UInt>::value, ""); - if (is_negative(value)) { - prefix[0] = '-'; - ++prefix_size; - abs_value = 0 - abs_value; - } else if (specs.sign != sign::none && specs.sign != sign::minus) { - prefix[0] = specs.sign == sign::plus ? '+' : ' '; - ++prefix_size; - } +template +FMT_CONSTEXPR auto make_write_int_arg(T value, sign_t sign) + -> write_int_arg> { + auto prefix = 0u; + auto abs_value = static_cast>(value); + if (is_negative(value)) { + prefix = 0x01000000 | '-'; + abs_value = 0 - abs_value; + } else { + prefix = data::prefixes[sign]; } + return {abs_value, prefix}; +} - void on_dec() { +template +FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg arg, + const basic_format_specs& specs, + locale_ref loc) -> OutputIt { + static_assert(std::is_same>::value, ""); + auto abs_value = arg.abs_value; + auto prefix = arg.prefix; + auto utype = static_cast(specs.type); + switch (specs.type) { + case 0: + case 'd': { + if (specs.localized && + write_int_localized(out, static_cast>(abs_value), + prefix, specs, loc)) { + return out; + } auto num_digits = count_digits(abs_value); - out = write_int( - out, num_digits, get_prefix(), specs, [this, num_digits](iterator it) { + return write_int( + out, num_digits, prefix, specs, [=](reserve_iterator it) { return format_decimal(it, abs_value, num_digits).end; }); } - - void on_hex() { - if (specs.alt) { - prefix[prefix_size++] = '0'; - prefix[prefix_size++] = specs.type; - } + case 'x': + case 'X': { + if (specs.alt) prefix_append(prefix, (utype << 8) | '0'); + bool upper = specs.type != 'x'; int num_digits = count_digits<4>(abs_value); - out = write_int(out, num_digits, get_prefix(), specs, - [this, num_digits](iterator it) { - return format_uint<4, Char>(it, abs_value, num_digits, - specs.type != 'x'); - }); + return write_int( + out, num_digits, prefix, specs, [=](reserve_iterator it) { + return format_uint<4, Char>(it, abs_value, num_digits, upper); + }); } - - void on_bin() { - if (specs.alt) { - prefix[prefix_size++] = '0'; - prefix[prefix_size++] = static_cast(specs.type); - } + case 'b': + case 'B': { + if (specs.alt) prefix_append(prefix, (utype << 8) | '0'); int num_digits = count_digits<1>(abs_value); - out = write_int(out, num_digits, get_prefix(), specs, - [this, num_digits](iterator it) { - return format_uint<1, Char>(it, abs_value, num_digits); - }); + return write_int(out, num_digits, prefix, specs, + [=](reserve_iterator it) { + return format_uint<1, Char>(it, abs_value, num_digits); + }); } - - void on_oct() { + case 'o': { int num_digits = count_digits<3>(abs_value); if (specs.alt && specs.precision <= num_digits && abs_value != 0) { // Octal prefix '0' is counted as a digit, so only add it if precision // is not greater than the number of digits. - prefix[prefix_size++] = '0'; - } - out = write_int(out, num_digits, get_prefix(), specs, - [this, num_digits](iterator it) { - return format_uint<3, Char>(it, abs_value, num_digits); - }); - } - - enum { sep_size = 1 }; - - void on_num() { - std::string groups = grouping(locale); - if (groups.empty()) return on_dec(); - auto sep = thousands_sep(locale); - if (!sep) return on_dec(); - int num_digits = count_digits(abs_value); - int size = num_digits, n = num_digits; - std::string::const_iterator group = groups.cbegin(); - while (group != groups.cend() && n > *group && *group > 0 && - *group != max_value()) { - size += sep_size; - n -= *group; - ++group; - } - if (group == groups.cend()) size += sep_size * ((n - 1) / groups.back()); - char digits[40]; - format_decimal(digits, abs_value, num_digits); - basic_memory_buffer buffer; - size += static_cast(prefix_size); - const auto usize = to_unsigned(size); - buffer.resize(usize); - basic_string_view s(&sep, sep_size); - // Index of a decimal digit with the least significant digit having index 0. - int digit_index = 0; - group = groups.cbegin(); - auto p = buffer.data() + size - 1; - for (int i = num_digits - 1; i > 0; --i) { - *p-- = static_cast(digits[i]); - if (*group <= 0 || ++digit_index % *group != 0 || - *group == max_value()) - continue; - if (group + 1 != groups.cend()) { - digit_index = 0; - ++group; - } - std::uninitialized_copy(s.data(), s.data() + s.size(), - make_checked(p, s.size())); - p -= s.size(); + prefix_append(prefix, '0'); } - *p-- = static_cast(*digits); - if (prefix_size != 0) *p = static_cast('-'); - auto data = buffer.data(); - out = write_padded( - out, specs, usize, usize, - [=](iterator it) { return copy_str(data, data + size, it); }); + return write_int(out, num_digits, prefix, specs, + [=](reserve_iterator it) { + return format_uint<3, Char>(it, abs_value, num_digits); + }); } - - void on_chr() { *out++ = static_cast(abs_value); } - - FMT_NORETURN void on_error() { + case 'c': + return write_char(out, static_cast(abs_value), specs); + default: FMT_THROW(format_error("invalid type specifier")); } -}; + return out; +} +template ::value && + !std::is_same::value && + std::is_same>::value)> +FMT_CONSTEXPR auto write(OutputIt out, T value, + const basic_format_specs& specs, locale_ref loc) + -> OutputIt { + return write_int(out, make_write_int_arg(value, specs.sign), specs, loc); +} +// An inlined version of write used in format string compilation. +template ::value && + !std::is_same::value && + !std::is_same>::value)> +FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value, + const basic_format_specs& specs, + locale_ref loc) -> OutputIt { + return write_int(out, make_write_int_arg(value, specs.sign), specs, loc); +} template -OutputIt write_nonfinite(OutputIt out, bool isinf, - const basic_format_specs& specs, - const float_specs& fspecs) { +FMT_CONSTEXPR auto write(OutputIt out, basic_string_view s, + const basic_format_specs& specs) -> OutputIt { + auto data = s.data(); + auto size = s.size(); + if (specs.precision >= 0 && to_unsigned(specs.precision) < size) + size = code_point_index(s, to_unsigned(specs.precision)); + auto width = + specs.width != 0 ? compute_width(basic_string_view(data, size)) : 0; + return write_padded(out, specs, size, width, + [=](reserve_iterator it) { + return copy_str(data, data + size, it); + }); +} +template +FMT_CONSTEXPR auto write(OutputIt out, + basic_string_view> s, + const basic_format_specs& specs, locale_ref) + -> OutputIt { + check_string_type_spec(specs.type); + return write(out, s, specs); +} +template +FMT_CONSTEXPR auto write(OutputIt out, const Char* s, + const basic_format_specs& specs, locale_ref) + -> OutputIt { + return check_cstring_type_spec(specs.type) + ? write(out, basic_string_view(s), specs, {}) + : write_ptr(out, to_uintptr(s), &specs); +} + +template +auto write_nonfinite(OutputIt out, bool isinf, basic_format_specs specs, + const float_specs& fspecs) -> OutputIt { auto str = isinf ? (fspecs.upper ? "INF" : "inf") : (fspecs.upper ? "NAN" : "nan"); constexpr size_t str_size = 3; auto sign = fspecs.sign; auto size = str_size + (sign ? 1 : 0); - using iterator = remove_reference_t; - return write_padded(out, specs, size, [=](iterator it) { + // Replace '0'-padding with space for non-finite values. + const bool is_zero_fill = + specs.fill.size() == 1 && *specs.fill.data() == static_cast('0'); + if (is_zero_fill) specs.fill[0] = static_cast(' '); + return write_padded(out, specs, size, [=](reserve_iterator it) { if (sign) *it++ = static_cast(data::signs[sign]); return copy_str(str, str + str_size, it); }); @@ -1738,74 +1606,76 @@ int exponent; }; -inline int get_significand_size(const big_decimal_fp& fp) { +inline auto get_significand_size(const big_decimal_fp& fp) -> int { return fp.significand_size; } template -inline int get_significand_size(const dragonbox::decimal_fp& fp) { +inline auto get_significand_size(const dragonbox::decimal_fp& fp) -> int { return count_digits(fp.significand); } template -inline OutputIt write_significand(OutputIt out, const char* significand, - int& significand_size) { +inline auto write_significand(OutputIt out, const char* significand, + int& significand_size) -> OutputIt { return copy_str(significand, significand + significand_size, out); } template -inline OutputIt write_significand(OutputIt out, UInt significand, - int significand_size) { +inline auto write_significand(OutputIt out, UInt significand, + int significand_size) -> OutputIt { return format_decimal(out, significand, significand_size).end; } template ::value)> -inline Char* write_significand(Char* out, UInt significand, - int significand_size, int integral_size, - Char decimal_point) { +inline auto write_significand(Char* out, UInt significand, int significand_size, + int integral_size, Char decimal_point) -> Char* { if (!decimal_point) return format_decimal(out, significand, significand_size).end; auto end = format_decimal(out + 1, significand, significand_size).end; - if (integral_size == 1) + if (integral_size == 1) { out[0] = out[1]; - else - std::copy_n(out + 1, integral_size, out); + } else { + std::uninitialized_copy_n(out + 1, integral_size, + make_checked(out, to_unsigned(integral_size))); + } out[integral_size] = decimal_point; return end; } template >::value)> -inline OutputIt write_significand(OutputIt out, UInt significand, - int significand_size, int integral_size, - Char decimal_point) { +inline auto write_significand(OutputIt out, UInt significand, + int significand_size, int integral_size, + Char decimal_point) -> OutputIt { // Buffer is large enough to hold digits (digits10 + 1) and a decimal point. Char buffer[digits10() + 2]; auto end = write_significand(buffer, significand, significand_size, integral_size, decimal_point); - return detail::copy_str(buffer, end, out); + return detail::copy_str_noinline(buffer, end, out); } template -inline OutputIt write_significand(OutputIt out, const char* significand, - int significand_size, int integral_size, - Char decimal_point) { - out = detail::copy_str(significand, significand + integral_size, out); +inline auto write_significand(OutputIt out, const char* significand, + int significand_size, int integral_size, + Char decimal_point) -> OutputIt { + out = detail::copy_str_noinline(significand, + significand + integral_size, out); if (!decimal_point) return out; *out++ = decimal_point; - return detail::copy_str(significand + integral_size, - significand + significand_size, out); + return detail::copy_str_noinline(significand + integral_size, + significand + significand_size, out); } template -OutputIt write_float(OutputIt out, const DecimalFP& fp, - const basic_format_specs& specs, float_specs fspecs, - Char decimal_point) { +auto write_float(OutputIt out, const DecimalFP& fp, + const basic_format_specs& specs, float_specs fspecs, + Char decimal_point) -> OutputIt { auto significand = fp.significand; int significand_size = get_significand_size(fp); static const Char zero = static_cast('0'); auto sign = fspecs.sign; size_t size = to_unsigned(significand_size) + (sign ? 1 : 0); - using iterator = remove_reference_t; + using iterator = reserve_iterator; int output_exp = fp.exponent + significand_size - 1; auto use_exp_format = [=]() { @@ -1820,7 +1690,8 @@ if (use_exp_format()) { int num_zeros = 0; if (fspecs.showpoint) { - num_zeros = (std::max)(fspecs.precision - significand_size, 0); + num_zeros = fspecs.precision - significand_size; + if (num_zeros < 0) num_zeros = 0; size += to_unsigned(num_zeros); } else if (significand_size == 1) { decimal_point = Char(); @@ -1836,7 +1707,7 @@ // Insert a decimal point after the first digit and add an exponent. it = write_significand(it, significand, significand_size, 1, decimal_point); - if (num_zeros > 0) it = std::fill_n(it, num_zeros, zero); + if (num_zeros > 0) it = detail::fill_n(it, num_zeros, zero); *it++ = static_cast(exp_char); return write_exponent(output_exp, it); }; @@ -1855,15 +1726,15 @@ #endif if (fspecs.showpoint) { if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 1; - if (num_zeros > 0) size += to_unsigned(num_zeros); + if (num_zeros > 0) size += to_unsigned(num_zeros) + 1; } return write_padded(out, specs, size, [&](iterator it) { if (sign) *it++ = static_cast(data::signs[sign]); it = write_significand(it, significand, significand_size); - it = std::fill_n(it, fp.exponent, zero); + it = detail::fill_n(it, fp.exponent, zero); if (!fspecs.showpoint) return it; *it++ = decimal_point; - return num_zeros > 0 ? std::fill_n(it, num_zeros, zero) : it; + return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it; }); } else if (exp > 0) { // 1234e-2 -> 12.34[0+] @@ -1873,7 +1744,7 @@ if (sign) *it++ = static_cast(data::signs[sign]); it = write_significand(it, significand, significand_size, exp, decimal_point); - return num_zeros > 0 ? std::fill_n(it, num_zeros, zero) : it; + return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it; }); } // 1234e-6 -> 0.001234 @@ -1882,21 +1753,22 @@ fspecs.precision < num_zeros) { num_zeros = fspecs.precision; } - size += 2 + to_unsigned(num_zeros); + bool pointy = num_zeros != 0 || significand_size != 0 || fspecs.showpoint; + size += 1 + (pointy ? 1 : 0) + to_unsigned(num_zeros); return write_padded(out, specs, size, [&](iterator it) { if (sign) *it++ = static_cast(data::signs[sign]); *it++ = zero; - if (num_zeros == 0 && significand_size == 0 && !fspecs.showpoint) return it; + if (!pointy) return it; *it++ = decimal_point; - it = std::fill_n(it, num_zeros, zero); + it = detail::fill_n(it, num_zeros, zero); return write_significand(it, significand, significand_size); }); } template ::value)> -OutputIt write(OutputIt out, T value, basic_format_specs specs, - locale_ref loc = {}) { +auto write(OutputIt out, T value, basic_format_specs specs, + locale_ref loc = {}) -> OutputIt { if (const_check(!is_supported_floating_point(value))) return out; float_specs fspecs = parse_float_type_spec(specs); fspecs.sign = specs.sign; @@ -1922,7 +1794,8 @@ if (fspecs.format == float_format::hex) { if (fspecs.sign) buffer.push_back(data::signs[fspecs.sign]); snprintf_float(promote_float(value), specs.precision, fspecs, buffer); - return write_bytes(out, {buffer.data(), buffer.size()}, specs); + return write_bytes(out, {buffer.data(), buffer.size()}, + specs); } int precision = specs.precision >= 0 || !specs.type ? specs.precision : 6; if (fspecs.format == float_format::exp) { @@ -1943,7 +1816,7 @@ template ::value)> -OutputIt write(OutputIt out, T value) { +auto write(OutputIt out, T value) -> OutputIt { if (const_check(!is_supported_floating_point(value))) return out; using floaty = conditional_t::value, double, T>; @@ -1969,72 +1842,36 @@ template ::value && !is_fast_float::value)> -inline OutputIt write(OutputIt out, T value) { +inline auto write(OutputIt out, T value) -> OutputIt { return write(out, value, basic_format_specs()); } template -OutputIt write_char(OutputIt out, Char value, - const basic_format_specs& specs) { - using iterator = remove_reference_t; - return write_padded(out, specs, 1, [=](iterator it) { - *it++ = value; - return it; - }); -} - -template -OutputIt write_ptr(OutputIt out, UIntPtr value, - const basic_format_specs* specs) { - int num_digits = count_digits<4>(value); - auto size = to_unsigned(num_digits) + size_t(2); - using iterator = remove_reference_t; - auto write = [=](iterator it) { - *it++ = static_cast('0'); - *it++ = static_cast('x'); - return format_uint<4, Char>(it, value, num_digits); - }; - return specs ? write_padded(out, *specs, size, write) - : base_iterator(out, write(reserve(out, size))); -} - -template struct is_integral : std::is_integral {}; -template <> struct is_integral : std::true_type {}; -template <> struct is_integral : std::true_type {}; - -template -OutputIt write(OutputIt out, monostate) { +auto write(OutputIt out, monostate, basic_format_specs = {}, + locale_ref = {}) -> OutputIt { FMT_ASSERT(false, ""); return out; } -template ::value)> -OutputIt write(OutputIt out, string_view value) { - auto it = reserve(out, value.size()); - it = copy_str(value.begin(), value.end(), it); - return base_iterator(out, it); -} - template -OutputIt write(OutputIt out, basic_string_view value) { +FMT_CONSTEXPR auto write(OutputIt out, basic_string_view value) + -> OutputIt { auto it = reserve(out, value.size()); - it = std::copy(value.begin(), value.end(), it); + it = copy_str_noinline(value.begin(), value.end(), it); return base_iterator(out, it); } -template -buffer_appender write(buffer_appender out, - basic_string_view value) { - get_container(out).append(value.begin(), value.end()); - return out; +template ::value)> +constexpr auto write(OutputIt out, const T& value) -> OutputIt { + return write(out, to_string_view(value)); } template ::value && !std::is_same::value && !std::is_same::value)> -OutputIt write(OutputIt out, T value) { +FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt { auto abs_value = static_cast>(value); bool negative = is_negative(value); // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer. @@ -2052,20 +1889,39 @@ return base_iterator(out, it); } -template -OutputIt write(OutputIt out, bool value) { - return write(out, string_view(value ? "true" : "false")); +// FMT_ENABLE_IF() condition separated to workaround MSVC bug +template < + typename Char, typename OutputIt, typename T, + bool check = + std::is_enum::value && !std::is_same::value && + mapped_type_constant>::value != + type::custom_type, + FMT_ENABLE_IF(check)> +FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt { + return write( + out, static_cast::type>(value)); +} + +template ::value)> +FMT_CONSTEXPR auto write(OutputIt out, T value, + const basic_format_specs& specs = {}, + locale_ref = {}) -> OutputIt { + return specs.type && specs.type != 's' + ? write(out, value ? 1 : 0, specs, {}) + : write_bytes(out, value ? "true" : "false", specs); } template -OutputIt write(OutputIt out, Char value) { +FMT_CONSTEXPR auto write(OutputIt out, Char value) -> OutputIt { auto it = reserve(out, 1); *it++ = value; return base_iterator(out, it); } template -OutputIt write(OutputIt out, const Char* value) { +FMT_CONSTEXPR_CHAR_TRAITS auto write(OutputIt out, const Char* value) + -> OutputIt { if (!value) { FMT_THROW(format_error("string pointer is null")); } else { @@ -2075,16 +1931,21 @@ return out; } -template -OutputIt write(OutputIt out, const void* value) { - return write_ptr(out, to_uintptr(value), nullptr); +template ::value)> +auto write(OutputIt out, const T* value, + const basic_format_specs& specs = {}, locale_ref = {}) + -> OutputIt { + check_pointer_type_spec(specs.type, error_handler()); + return write_ptr(out, to_uintptr(value), &specs); } template -auto write(OutputIt out, const T& value) -> typename std::enable_if< - mapped_type_constant>::value == - type::custom_type, - OutputIt>::type { +FMT_CONSTEXPR auto write(OutputIt out, const T& value) -> + typename std::enable_if< + mapped_type_constant>::value == + type::custom_type, + OutputIt>::type { using context_type = basic_format_context; using formatter_type = conditional_t::value, @@ -2096,292 +1957,52 @@ // An argument visitor that formats the argument and writes it via the output // iterator. It's a class and not a generic lambda for compatibility with C++11. -template struct default_arg_formatter { - using context = basic_format_context; +template struct default_arg_formatter { + using iterator = buffer_appender; + using context = buffer_context; - OutputIt out; + iterator out; basic_format_args args; locale_ref loc; - template OutputIt operator()(T value) { + template auto operator()(T value) -> iterator { return write(out, value); } - - OutputIt operator()(typename basic_format_arg::handle handle) { + auto operator()(typename basic_format_arg::handle h) -> iterator { basic_format_parse_context parse_ctx({}); - basic_format_context format_ctx(out, args, loc); - handle.format(parse_ctx, format_ctx); + context format_ctx(out, args, loc); + h.format(parse_ctx, format_ctx); return format_ctx.out(); } }; -template -class arg_formatter_base { - public: - using iterator = OutputIt; - using char_type = Char; - using format_specs = basic_format_specs; - - private: - iterator out_; - locale_ref locale_; - format_specs* specs_; - - // Attempts to reserve space for n extra characters in the output range. - // Returns a pointer to the reserved range or a reference to out_. - auto reserve(size_t n) -> decltype(detail::reserve(out_, n)) { - return detail::reserve(out_, n); - } - - using reserve_iterator = remove_reference_t(), 0))>; - - template void write_int(T value, const format_specs& spec) { - using uint_type = uint32_or_64_or_128_t; - int_writer w(out_, locale_, value, spec); - handle_int_type_spec(spec.type, w); - out_ = w.out; - } - - void write(char value) { - auto&& it = reserve(1); - *it++ = value; - } - - template ::value)> - void write(Ch value) { - out_ = detail::write(out_, value); - } - - void write(string_view value) { - auto&& it = reserve(value.size()); - it = copy_str(value.begin(), value.end(), it); - } - void write(wstring_view value) { - static_assert(std::is_same::value, ""); - auto&& it = reserve(value.size()); - it = std::copy(value.begin(), value.end(), it); - } - - template - void write(const Ch* s, size_t size, const format_specs& specs) { - auto width = specs.width != 0 - ? count_code_points(basic_string_view(s, size)) - : 0; - out_ = write_padded(out_, specs, size, width, [=](reserve_iterator it) { - return copy_str(s, s + size, it); - }); - } - - template - void write(basic_string_view s, const format_specs& specs = {}) { - out_ = detail::write(out_, s, specs); - } - - void write_pointer(const void* p) { - out_ = write_ptr(out_, to_uintptr(p), specs_); - } - - struct char_spec_handler : ErrorHandler { - arg_formatter_base& formatter; - Char value; - - char_spec_handler(arg_formatter_base& f, Char val) - : formatter(f), value(val) {} - - void on_int() { - // char is only formatted as int if there are specs. - formatter.write_int(static_cast(value), *formatter.specs_); - } - void on_char() { - if (formatter.specs_) - formatter.out_ = write_char(formatter.out_, value, *formatter.specs_); - else - formatter.write(value); - } - }; - - struct cstring_spec_handler : error_handler { - arg_formatter_base& formatter; - const Char* value; - - cstring_spec_handler(arg_formatter_base& f, const Char* val) - : formatter(f), value(val) {} - - void on_string() { formatter.write(value); } - void on_pointer() { formatter.write_pointer(value); } - }; - - protected: - iterator out() { return out_; } - format_specs* specs() { return specs_; } - - void write(bool value) { - if (specs_) - write(string_view(value ? "true" : "false"), *specs_); - else - out_ = detail::write(out_, value); - } - - void write(const Char* value) { - if (!value) { - FMT_THROW(format_error("string pointer is null")); - } else { - auto length = std::char_traits::length(value); - basic_string_view sv(value, length); - specs_ ? write(sv, *specs_) : write(sv); - } - } - - public: - arg_formatter_base(OutputIt out, format_specs* s, locale_ref loc) - : out_(out), locale_(loc), specs_(s) {} - - iterator operator()(monostate) { - FMT_ASSERT(false, "invalid argument type"); - return out_; - } - - template ::value)> - FMT_INLINE iterator operator()(T value) { - if (specs_) - write_int(value, *specs_); - else - out_ = detail::write(out_, value); - return out_; - } - - iterator operator()(Char value) { - handle_char_specs(specs_, - char_spec_handler(*this, static_cast(value))); - return out_; - } - - iterator operator()(bool value) { - if (specs_ && specs_->type) return (*this)(value ? 1 : 0); - write(value != 0); - return out_; - } - - template ::value)> - iterator operator()(T value) { - auto specs = specs_ ? *specs_ : format_specs(); - if (const_check(is_supported_floating_point(value))) - out_ = detail::write(out_, value, specs, locale_); - else - FMT_ASSERT(false, "unsupported float argument type"); - return out_; - } - - iterator operator()(const Char* value) { - if (!specs_) return write(value), out_; - handle_cstring_type_spec(specs_->type, cstring_spec_handler(*this, value)); - return out_; - } - - iterator operator()(basic_string_view value) { - if (specs_) { - check_string_type_spec(specs_->type, error_handler()); - write(value, *specs_); - } else { - write(value); - } - return out_; - } - - iterator operator()(const void* value) { - if (specs_) check_pointer_type_spec(specs_->type, error_handler()); - write_pointer(value); - return out_; - } -}; - -/** The default argument formatter. */ -template -class arg_formatter : public arg_formatter_base { - private: - using char_type = Char; - using base = arg_formatter_base; - using context_type = basic_format_context; - - context_type& ctx_; - basic_format_parse_context* parse_ctx_; - const Char* ptr_; +template struct arg_formatter { + using iterator = buffer_appender; + using context = buffer_context; - public: - using iterator = typename base::iterator; - using format_specs = typename base::format_specs; + iterator out; + const basic_format_specs& specs; + locale_ref locale; - /** - \rst - Constructs an argument formatter object. - *ctx* is a reference to the formatting context, - *specs* contains format specifier information for standard argument types. - \endrst - */ - explicit arg_formatter( - context_type& ctx, - basic_format_parse_context* parse_ctx = nullptr, - format_specs* specs = nullptr, const Char* ptr = nullptr) - : base(ctx.out(), specs, ctx.locale()), - ctx_(ctx), - parse_ctx_(parse_ctx), - ptr_(ptr) {} - - using base::operator(); - - /** Formats an argument of a user-defined type. */ - iterator operator()(typename basic_format_arg::handle handle) { - if (ptr_) advance_to(*parse_ctx_, ptr_); - handle.format(*parse_ctx_, ctx_); - return ctx_.out(); + template + FMT_CONSTEXPR FMT_INLINE auto operator()(T value) -> iterator { + return detail::write(out, value, specs, locale); + } + auto operator()(typename basic_format_arg::handle) -> iterator { + // User-defined types are handled separately because they require access + // to the parse context. + return out; } }; -template FMT_CONSTEXPR bool is_name_start(Char c) { - return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || '_' == c; -} - -// Parses the range [begin, end) as an unsigned integer. This function assumes -// that the range is non-empty and the first character is a digit. -template -FMT_CONSTEXPR int parse_nonnegative_int(const Char*& begin, const Char* end, - ErrorHandler&& eh) { - FMT_ASSERT(begin != end && '0' <= *begin && *begin <= '9', ""); - unsigned value = 0; - // Convert to unsigned to prevent a warning. - constexpr unsigned max_int = max_value(); - unsigned big = max_int / 10; - do { - // Check for overflow. - if (value > big) { - value = max_int + 1; - break; - } - value = value * 10 + unsigned(*begin - '0'); - ++begin; - } while (begin != end && '0' <= *begin && *begin <= '9'); - if (value > max_int) eh.on_error("number is too big"); - return static_cast(value); -} - -template class custom_formatter { - private: - using char_type = typename Context::char_type; - - basic_format_parse_context& parse_ctx_; - Context& ctx_; - - public: - explicit custom_formatter(basic_format_parse_context& parse_ctx, - Context& ctx) - : parse_ctx_(parse_ctx), ctx_(ctx) {} - - void operator()(typename basic_format_arg::handle h) const { - h.format(parse_ctx_, ctx_); +template struct custom_formatter { + basic_format_parse_context& parse_ctx; + buffer_context& ctx; + + void operator()( + typename basic_format_arg>::handle h) const { + h.format(parse_ctx, ctx); } - template void operator()(T) const {} }; @@ -2396,13 +2017,13 @@ explicit FMT_CONSTEXPR width_checker(ErrorHandler& eh) : handler_(eh) {} template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T value) { + FMT_CONSTEXPR auto operator()(T value) -> unsigned long long { if (is_negative(value)) handler_.on_error("negative width"); return static_cast(value); } template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T) { + FMT_CONSTEXPR auto operator()(T) -> unsigned long long { handler_.on_error("width is not integer"); return 0; } @@ -2416,13 +2037,13 @@ explicit FMT_CONSTEXPR precision_checker(ErrorHandler& eh) : handler_(eh) {} template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T value) { + FMT_CONSTEXPR auto operator()(T value) -> unsigned long long { if (is_negative(value)) handler_.on_error("negative precision"); return static_cast(value); } template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T) { + FMT_CONSTEXPR auto operator()(T) -> unsigned long long { handler_.on_error("precision is not integer"); return 0; } @@ -2431,152 +2052,54 @@ ErrorHandler& handler_; }; -// A format specifier handler that sets fields in basic_format_specs. -template class specs_setter { - public: - explicit FMT_CONSTEXPR specs_setter(basic_format_specs& specs) - : specs_(specs) {} +template