[Cmake-commits] CMake branch, next, updated. v3.7.0-rc1-594-g387e436
Brad King
brad.king at kitware.com
Tue Oct 18 11:20:06 EDT 2016
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CMake".
The branch, next has been updated
via 387e43639e56691356a9c5617b24ff7ff7e25e92 (commit)
via 33a9aaa89b437da29a70f27d25b267242b6f982c (commit)
from 212e620fcb657ec83b8ba3f469dccf6a0b693102 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=387e43639e56691356a9c5617b24ff7ff7e25e92
commit 387e43639e56691356a9c5617b24ff7ff7e25e92
Merge: 212e620 33a9aaa
Author: Brad King <brad.king at kitware.com>
AuthorDate: Tue Oct 18 11:20:03 2016 -0400
Commit: CMake Topic Stage <kwrobot at kitware.com>
CommitDate: Tue Oct 18 11:20:03 2016 -0400
Merge topic 'ExternalData-multiple-hashes' into next
33a9aaa8 ExternalData: Add support for multiple hash algorithms
https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=33a9aaa89b437da29a70f27d25b267242b6f982c
commit 33a9aaa89b437da29a70f27d25b267242b6f982c
Author: Matt McCormick <matt.mccormick at kitware.com>
AuthorDate: Wed Oct 12 14:57:32 2016 -0400
Commit: Brad King <brad.king at kitware.com>
CommitDate: Tue Oct 18 11:16:45 2016 -0400
ExternalData: Add support for multiple hash algorithms
Add support for projects to have `Data.txt.md5` *and* `Data.txt.sha512`
where the content links hold hashes for the same file. Check all
`ExternalData_URL_TEMPLATES` entries in order for all available hashes.
The data acquisition is considered a failure if none of the available
URL resources has any of the given hashes. This makes it possible to
have multiple data server resources where all servers do not support all
hashing algorithms.
diff --git a/Help/release/dev/ExternalData-multiple-hashes.rst b/Help/release/dev/ExternalData-multiple-hashes.rst
new file mode 100644
index 0000000..608a277
--- /dev/null
+++ b/Help/release/dev/ExternalData-multiple-hashes.rst
@@ -0,0 +1,7 @@
+ExternalData-multiple-hashes
+----------------------------
+
+* The :module:`ExternalData` module learned to support multiple
+ content links for one data file using different hashes, e.g.
+ ``img.png.sha256`` and ``img.png.sha1``. This allows objects
+ to be fetched from sources indexed by different hash algorithms.
diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake
index b00de14..07cd114 100644
--- a/Modules/ExternalData.cmake
+++ b/Modules/ExternalData.cmake
@@ -201,6 +201,11 @@ For example, the argument ``DATA{img.png}`` may be satisfied by either a
real ``img.png`` file in the current source directory or a ``img.png.md5``
file containing its MD5 sum.
+Multiple content links of the same name with different hash algorithms
+are supported (e.g. ``img.png.sha256`` and ``img.png.sha1``) so long as
+they all correspond to the same real file. This allows objects to be
+fetched from sources indexed by different hash algorithms.
+
Referencing File Series
"""""""""""""""""""""""
@@ -429,8 +434,10 @@ function(ExternalData_add_target target)
string(REPLACE "|" ";" tuple "${entry}")
list(GET tuple 0 file)
list(GET tuple 1 name)
- list(GET tuple 2 ext)
- set(stamp "${ext}-stamp")
+ list(GET tuple 2 exts)
+ string(REPLACE "+" ";" exts_list "${exts}")
+ list(GET exts_list 0 first_ext)
+ set(stamp "-hash-stamp")
if(NOT DEFINED "_ExternalData_FILE_${file}")
set("_ExternalData_FILE_${file}" 1)
get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}")
@@ -446,12 +453,12 @@ function(ExternalData_add_target target)
OUTPUT "${file}${stamp}" "${file}"
# Run the data fetch/update script.
COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR}
- -Dfile=${file} -Dname=${name} -Dext=${ext}
+ -Dfile=${file} -Dname=${name} -Dexts=${exts}
-DExternalData_ACTION=fetch
-DExternalData_CONFIG=${config}
-P ${_ExternalData_SELF}
# Update whenever the object hash changes.
- MAIN_DEPENDENCY "${name}${ext}"
+ MAIN_DEPENDENCY "${name}${first_ext}"
)
endif()
list(APPEND files "${file}${stamp}")
@@ -798,6 +805,7 @@ function(_ExternalData_arg_find_files glob pattern regex)
cmake_policy(SET CMP0009 NEW)
file(${glob} globbed RELATIVE "${top_src}" "${top_src}/${pattern}*")
cmake_policy(POP)
+ set(externals_count -1)
foreach(entry IN LISTS globbed)
if("x${entry}" MATCHES "^x(.*)(\\.(${_ExternalData_REGEX_EXT}))$")
set(relname "${CMAKE_MATCH_1}")
@@ -817,7 +825,11 @@ function(_ExternalData_arg_find_files glob pattern regex)
set(name "${top_src}/${relname}")
set(file "${top_bin}/${relname}")
if(alg)
- list(APPEND external "${file}|${name}|${alg}")
+ if(NOT "${external_${externals_count}_file_name}" STREQUAL "${file}|${name}")
+ math(EXPR externals_count "${externals_count} + 1")
+ set(external_${externals_count}_file_name "${file}|${name}")
+ endif()
+ list(APPEND external_${externals_count}_algs "${alg}")
elseif(ExternalData_LINK_CONTENT)
_ExternalData_link_content("${name}" alg)
list(APPEND external "${file}|${name}|${alg}")
@@ -830,6 +842,14 @@ function(_ExternalData_arg_find_files glob pattern regex)
endif()
endif()
endforeach()
+ if(${externals_count} GREATER -1)
+ foreach(ii RANGE ${externals_count})
+ string(REPLACE ";" "+" algs_delim "${external_${ii}_algs}")
+ list(APPEND external "${external_${ii}_file_name}|${algs_delim}")
+ unset(external_${ii}_algs)
+ unset(external_${ii}_file_name)
+ endforeach()
+ endif()
set(external "${external}" PARENT_SCOPE)
set(internal "${internal}" PARENT_SCOPE)
set(have_original "${have_original}" PARENT_SCOPE)
@@ -947,13 +967,28 @@ function(_ExternalData_custom_fetch key loc file err_var msg_var)
set("${msg_var}" "${msg}" PARENT_SCOPE)
endfunction()
-function(_ExternalData_download_object name hash algo var_obj)
+function(_ExternalData_get_from_object_store hash algo var_obj var_success)
+ # Search all object stores for an existing object.
+ foreach(dir ${ExternalData_OBJECT_STORES})
+ set(obj "${dir}/${algo}/${hash}")
+ if(EXISTS "${obj}")
+ message(STATUS "Found object: \"${obj}\"")
+ set("${var_obj}" "${obj}" PARENT_SCOPE)
+ set("${var_success}" 1 PARENT_SCOPE)
+ return()
+ endif()
+ endforeach()
+endfunction()
+
+function(_ExternalData_download_object name hash algo var_obj var_success var_errorMsg)
# Search all object stores for an existing object.
+ set(success 1)
foreach(dir ${ExternalData_OBJECT_STORES})
set(obj "${dir}/${algo}/${hash}")
if(EXISTS "${obj}")
message(STATUS "Found object: \"${obj}\"")
set("${var_obj}" "${obj}" PARENT_SCOPE)
+ set("${var_success}" "${success}" PARENT_SCOPE)
return()
endif()
endforeach()
@@ -1008,6 +1043,7 @@ function(_ExternalData_download_object name hash algo var_obj)
get_filename_component(dir "${name}" PATH)
set(staged "${dir}/.ExternalData_${algo}_${hash}")
+ set(success 1)
if(found)
file(RENAME "${tmp}" "${obj}")
message(STATUS "Downloaded object: \"${obj}\"")
@@ -1018,38 +1054,74 @@ function(_ExternalData_download_object name hash algo var_obj)
if(NOT tried)
set(tried "\n (No ExternalData_URL_TEMPLATES given)")
endif()
- message(FATAL_ERROR "Object ${algo}=${hash} not found at:${tried}")
+ set(success 0)
+ set("${var_errorMsg}" "Object ${algo}=${hash} not found at:${tried}" PARENT_SCOPE)
endif()
set("${var_obj}" "${obj}" PARENT_SCOPE)
+ set("${var_success}" "${success}" PARENT_SCOPE)
endfunction()
if("${ExternalData_ACTION}" STREQUAL "fetch")
- foreach(v ExternalData_OBJECT_STORES file name ext)
+ foreach(v ExternalData_OBJECT_STORES file name exts)
if(NOT DEFINED "${v}")
message(FATAL_ERROR "No \"-D${v}=\" value provided!")
endif()
endforeach()
- file(READ "${name}${ext}" hash)
- string(STRIP "${hash}" hash)
-
- if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
- string(TOUPPER "${CMAKE_MATCH_1}" algo)
- else()
- message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
- endif()
+ string(REPLACE "+" ";" exts_list "${exts}")
+ set(succeeded 0)
+ set(errorMsg "")
+ set(hash_list )
+ set(algo_list )
+ set(hash )
+ set(algo )
+ foreach(ext ${exts_list})
+ file(READ "${name}${ext}" hash)
+ string(STRIP "${hash}" hash)
+
+ if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
+ string(TOUPPER "${CMAKE_MATCH_1}" algo)
+ else()
+ message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
+ endif()
- _ExternalData_download_object("${name}" "${hash}" "${algo}" obj)
+ list(APPEND hash_list ${hash})
+ list(APPEND algo_list ${algo})
+ endforeach()
+ list(LENGTH exts_list num_extensions)
+ math(EXPR exts_range "${num_extensions} - 1")
+ foreach(ii RANGE 0 ${exts_range})
+ list(GET hash_list ${ii} hash)
+ list(GET algo_list ${ii} algo)
+ _ExternalData_get_from_object_store("${hash}" "${algo}" obj succeeded)
+ if(succeeded)
+ break()
+ endif()
+ endforeach()
+ if(NOT succeeded)
+ foreach(ii RANGE 0 ${exts_range})
+ list(GET hash_list ${ii} hash)
+ list(GET algo_list ${ii} algo)
+ _ExternalData_download_object("${name}" "${hash}" "${algo}"
+ obj succeeded algoErrorMsg)
+ set(errorMsg "${errorMsg}\n${algoErrorMsg}")
+ if(succeeded)
+ break()
+ endif()
+ endforeach()
+ endif()
+ if(NOT succeeded)
+ message(FATAL_ERROR "${errorMsg}")
+ endif()
# Check if file already corresponds to the object.
- set(stamp "${ext}-stamp")
+ set(stamp "-hash-stamp")
set(file_up_to_date 0)
if(EXISTS "${file}" AND EXISTS "${file}${stamp}")
file(READ "${file}${stamp}" f_hash)
string(STRIP "${f_hash}" f_hash)
if("${f_hash}" STREQUAL "${hash}")
- #message(STATUS "File already corresponds to object")
set(file_up_to_date 1)
endif()
endif()
diff --git a/Tests/Module/ExternalData/CMakeLists.txt b/Tests/Module/ExternalData/CMakeLists.txt
index 1018dd8..737e17a 100644
--- a/Tests/Module/ExternalData/CMakeLists.txt
+++ b/Tests/Module/ExternalData/CMakeLists.txt
@@ -45,6 +45,8 @@ ExternalData_Add_Test(Data1
-D Meta=DATA{MetaTop.dat,REGEX:Meta[ABC].dat}
-D Directory=DATA{Directory/,A.dat,REGEX:[BC].dat}
-D DirRecurse=DATA{DirRecurse/,RECURSE:,A.dat,REGEX:[BC].dat}
+ -D MultipleAlgorithmNoSHA1=DATA{MultipleAlgorithmNoSHA1.dat}
+ -D MultipleAlgorithmNoMD5=DATA{MultipleAlgorithmNoMD5.dat}
-D "Semicolons=DATA{Data.dat}\\;DATA{Data.dat}"
-P ${CMAKE_CURRENT_SOURCE_DIR}/Data1Check.cmake
)
diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5 b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5
new file mode 100644
index 0000000..a956f36
--- /dev/null
+++ b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5
@@ -0,0 +1 @@
+29848e54a4d0343f138ab14419b863de
diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.sha1 b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.sha1
new file mode 100644
index 0000000..43a3540
--- /dev/null
+++ b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.sha1
@@ -0,0 +1 @@
+2af59a7022024974f3b8521b7ed8137c996a79f1
diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.md5 b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.md5
new file mode 100644
index 0000000..1906cbf
--- /dev/null
+++ b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.md5
@@ -0,0 +1 @@
+08cfcf221f76ace7b906b312284e73d7
diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.sha1 b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.sha1
new file mode 100644
index 0000000..65781b2
--- /dev/null
+++ b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.sha1
@@ -0,0 +1 @@
+223b134e6e3a9bf34aa7531c009d97cff6b0d8a3
-----------------------------------------------------------------------
Summary of changes:
Help/release/dev/ExternalData-multiple-hashes.rst | 7 ++
Modules/ExternalData.cmake | 110 ++++++++++++++++----
Tests/Module/ExternalData/CMakeLists.txt | 2 +
.../ExternalData/MultipleAlgorithmNoMD5.dat.md5 | 1 +
....2.dat.sha1 => MultipleAlgorithmNoMD5.dat.sha1} | 0
...Top.dat.md5 => MultipleAlgorithmNoSHA1.dat.md5} | 0
.../ExternalData/MultipleAlgorithmNoSHA1.dat.sha1 | 1 +
7 files changed, 102 insertions(+), 19 deletions(-)
create mode 100644 Help/release/dev/ExternalData-multiple-hashes.rst
create mode 100644 Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5
copy Tests/Module/ExternalData/{SeriesMixed.2.dat.sha1 => MultipleAlgorithmNoMD5.dat.sha1} (100%)
copy Tests/Module/ExternalData/{MetaTop.dat.md5 => MultipleAlgorithmNoSHA1.dat.md5} (100%)
create mode 100644 Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.sha1
hooks/post-receive
--
CMake
More information about the Cmake-commits
mailing list