Ticket #5551: add_path.cpp

File add_path.cpp, 19.8 KB (added by David Willis <dawillis@…>, 11 years ago)

tools/bcp/add_path.cpp

Line 
1/*
2 *
3 * Copyright (c) 2003 Dr John Maddock
4 * Use, modification and distribution is subject to the
5 * Boost Software License, Version 1.0. (See accompanying file
6 * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7 *
8 * This file implements the following:
9 * void bcp_implementation::add_path(const fs::path& p)
10 * void bcp_implementation::add_directory(const fs::path& p)
11 * void bcp_implementation::add_file(const fs::path& p)
12 * void bcp_implementation::add_dependent_lib(const std::string& libname, const fs::path& p, const fileview& view)
13 */
14
15#include "bcp_imp.hpp"
16#include "fileview.hpp"
17#include <boost/regex.hpp>
18#include <boost/filesystem/operations.hpp>
19#include <boost/filesystem/exception.hpp>
20#include <iostream>
21
22
23void bcp_implementation::add_path(const fs::path& p)
24{
25 fs::path normalized_path = p;
26 normalized_path.normalize();
27 if(fs::exists(m_boost_path / normalized_path))
28 {
29 if(fs::is_directory(m_boost_path / normalized_path))
30 add_directory(normalized_path);
31 else
32 add_file(normalized_path);
33 }
34 else
35 {
36 std::cerr << "CAUTION: dependent file " << p.string() << " does not exist." << std::endl;
37 std::cerr << " Found while scanning file " << m_dependencies[p].string() << std::endl;
38 }
39}
40
41void bcp_implementation::add_directory(const fs::path& p)
42{
43 //
44 // Don't add files created by build system:
45 //
46 if((p.leaf() == "bin") || (p.leaf() == "bin-stage"))
47 return;
48 //
49 // Don't add version control directories:
50 //
51 if((p.leaf() == "CVS") || (p.leaf() == ".svn"))
52 return;
53 //
54 // don't add directories not under version control:
55 //
56 if(m_cvs_mode && !fs::exists(m_boost_path / p / "CVS/Entries"))
57 return;
58 if(m_svn_mode && !fs::exists(m_boost_path / p / ".svn/entries"))
59 return;
60 //
61 // enermerate files and directories:
62 //
63 fs::directory_iterator i(m_boost_path / p);
64 fs::directory_iterator j;
65 while(i != j)
66 {
67 //
68 // we need to convert *i back into
69 // a relative path, what follows is a hack:
70 //
71 std::string s(i->path().string());
72 if(m_boost_path.string().size())
73 s.erase(0, m_boost_path.string().size() + 1);
74 fs::path np = s;
75 if(!m_dependencies.count(np))
76 {
77 m_dependencies[np] = p; // set up dependency tree
78 add_path(np);
79 }
80 ++i;
81 }
82}
83
84void bcp_implementation::add_file(const fs::path& p)
85{
86 //
87 // if the file does not exist in cvs then don't do anything with it:
88 //
89 if((m_cvs_mode || m_svn_mode) && (m_cvs_paths.find(p) == m_cvs_paths.end()))
90 return;
91 //
92 // if we've already seen the file return:
93 //
94 if(m_copy_paths.find(p) != m_copy_paths.end())
95 return;
96 //
97 // add the file to our list:
98 //
99 m_copy_paths.insert(p);
100 //
101 // if this is a source file, scan for dependencies:
102 //
103 if(is_source_file(p))
104 {
105 add_file_dependencies(p, false);
106 }
107 if(is_jam_file(p) && m_namespace_name.size() && ((std::distance(p.begin(), p.end()) < 3) || (*p.begin() != "tools") || (*++p.begin() != "build")))
108 {
109 //
110 // We're doing a rename of namespaces and library names
111 // so scan for names of libraries:
112 //
113 static const boost::regex e(
114 "\\<lib\\s+(boost\\w+)\\s+[:;]"
115 );
116
117 fileview view(m_boost_path / p);
118 boost::regex_token_iterator<const char*> i(view.begin(), view.end(), e, 1);
119 boost::regex_token_iterator<const char*> j;
120 while(i != j)
121 {
122 m_lib_names.insert(*i);
123 ++i;
124 }
125 static const std::pair<fs::path, std::string> specials_library_names[] = {
126 std::pair<fs::path, std::string>("libs/python/build/Jamfile.v2", "boost_python"),
127 std::pair<fs::path, std::string>("libs/python/build/Jamfile.v2", "boost_python3"),
128 };
129
130 for(unsigned int n = 0; n < (sizeof(specials_library_names)/sizeof(specials_library_names[0])); ++n)
131 {
132 if(0 == compare_paths(specials_library_names[n].first, p))
133 {
134 m_lib_names.insert(specials_library_names[n].second);
135 }
136 }
137 }
138 //
139 // if this is a html file, scan for dependencies:
140 //
141 if(is_html_file(p))
142 {
143 static const boost::regex e(
144 "<(?:img[^>]*src=|link[^>]*href=)(\"[^\"]+\"|\\w+)[^>]*>"
145 );
146
147 fileview view(m_boost_path / p);
148 boost::regex_token_iterator<const char*> i(view.begin(), view.end(), e, 1);
149 boost::regex_token_iterator<const char*> j;
150 while(i != j)
151 {
152 //
153 // extract the dependent name:
154 //
155 std::string s(*i);
156 if(s[0] == '\"')
157 {
158 // remove quotes:
159 assert(s.size() > 2);
160 s.erase(0, 1);
161 s.erase(s.size() - 1);
162 }
163 //
164 // Remove any target suffix:
165 //
166 std::string::size_type n = s.find('#');
167 if(n != std::string::npos)
168 {
169 s.erase(n);
170 }
171 //
172 // if the name starts with ./ remove it
173 // or we'll get an error:
174 if(s.compare(0, 2, "./") == 0)
175 s.erase(0, 2);
176 if(s.find(':') == std::string::npos)
177 {
178 // only concatonate if it's a relative path
179 // rather than a URL:
180 fs::path dep(p.branch_path() / s);
181 if(!m_dependencies.count(dep))
182 {
183 m_dependencies[dep] = p; // set up dependency tree
184 add_path(dep);
185 }
186 }
187 ++i;
188 }
189 }
190 //
191 // now scan for "special" dependencies:
192 // anything that we can't automatically detect...
193 //
194static const std::pair<fs::path, fs::path>
195 specials[] = {
196 std::pair<fs::path, fs::path>("boost/config.hpp", "boost/config"),
197 std::pair<fs::path, fs::path>("tools/build/allyourbase.jam", "Jamrules"),
198 std::pair<fs::path, fs::path>("tools/build/allyourbase.jam", "project-root.jam"),
199 std::pair<fs::path, fs::path>("tools/build/allyourbase.jam", "boost-build.jam"),
200 std::pair<fs::path, fs::path>("tools/build/v1/allyourbase.jam", "Jamrules"),
201 std::pair<fs::path, fs::path>("tools/build/v1/allyourbase.jam", "project-root.jam"),
202 std::pair<fs::path, fs::path>("tools/build/v1/allyourbase.jam", "boost-build.jam"),
203 std::pair<fs::path, fs::path>("tools/build/v2/boost-build.jam", "Jamrules"),
204 std::pair<fs::path, fs::path>("tools/build/v2/boost-build.jam", "project-root.jam"),
205 std::pair<fs::path, fs::path>("tools/build/v2/boost-build.jam", "boost-build.jam"),
206 std::pair<fs::path, fs::path>("tools/build/v2/boost-build.jam", "Jamfile.v2"),
207 std::pair<fs::path, fs::path>("boost/preprocessor/iterate.hpp", "boost/preprocessor/iteration"),
208 std::pair<fs::path, fs::path>("boost/preprocessor/slot/slot.hpp", "boost/preprocessor/slot/detail"),
209 std::pair<fs::path, fs::path>("boost/function.hpp", "boost/function/detail"),
210 std::pair<fs::path, fs::path>("boost/regex/config.hpp", "boost/regex/user.hpp"),
211 std::pair<fs::path, fs::path>("boost/signals/signal_template.hpp", "boost/function"),
212 std::pair<fs::path, fs::path>("boost/mpl/list.hpp", "boost/mpl/list"),
213 std::pair<fs::path, fs::path>("boost/mpl/list_c.hpp", "boost/mpl/list"),
214 std::pair<fs::path, fs::path>("boost/mpl/vector.hpp", "boost/mpl/vector"),
215 std::pair<fs::path, fs::path>("boost/mpl/deque.hpp", "boost/mpl/vector"),
216 std::pair<fs::path, fs::path>("boost/mpl/vector_c.hpp", "boost/mpl/vector"),
217 std::pair<fs::path, fs::path>("boost/mpl/map.hpp", "boost/mpl/map"),
218 std::pair<fs::path, fs::path>("boost/mpl/set.hpp", "boost/mpl/set"),
219 std::pair<fs::path, fs::path>("boost/mpl/set_c.hpp", "boost/mpl/set"),
220 std::pair<fs::path, fs::path>("boost/mpl/aux_/include_preprocessed.hpp", "boost/mpl/aux_/preprocessed"),
221 std::pair<fs::path, fs::path>("boost/mpl/vector/aux_/include_preprocessed.hpp", "boost/mpl/vector/aux_/preprocessed"),
222 std::pair<fs::path, fs::path>("boost/mpl/set/aux_/include_preprocessed.hpp", "boost/mpl/set/aux_/preprocessed"),
223 std::pair<fs::path, fs::path>("boost/mpl/map/aux_/include_preprocessed.hpp", "boost/mpl/map/aux_/preprocessed"),
224 std::pair<fs::path, fs::path>("boost/mpl/list/aux_/include_preprocessed.hpp", "boost/mpl/list/aux_/preprocessed"),
225 std::pair<fs::path, fs::path>("libs/graph/src/python/visitor.hpp", "libs/graph/src/python"),
226 std::pair<fs::path, fs::path>("boost/test/detail/config.hpp", "libs/test/src"),
227 std::pair<fs::path, fs::path>("boost/test/detail/config.hpp", "libs/test/build"),
228 std::pair<fs::path, fs::path>("boost/typeof.hpp", "boost/typeof/incr_registration_group.hpp"),
229 std::pair<fs::path, fs::path>("boost/function_types/detail/pp_loop.hpp", "boost/function_types/detail/pp_cc_loop"),
230 std::pair<fs::path, fs::path>("boost/function_types/components.hpp", "boost/function_types/detail/components_impl"),
231 std::pair<fs::path, fs::path>("boost/function_types/detail/pp_loop.hpp", "boost/function_types/detail"),
232 std::pair<fs::path, fs::path>("boost/math/tools/rational.hpp", "boost/math/tools/detail"),
233 std::pair<fs::path, fs::path>("boost/proto/repeat.hpp", "boost/proto/detail/local.hpp"),
234 std::pair<fs::path, fs::path>("boost/signals/signal_template.hpp", "boost/function"),
235 std::pair<fs::path, fs::path>("boost/preprocessor/slot/counter.hpp", "boost/preprocessor/slot/detail/counter.hpp"),
236 std::pair<fs::path, fs::path>("boost/graph/distributed/detail/tag_allocator.hpp", "libs/graph_parallel"),
237 std::pair<fs::path, fs::path>("boost/graph/distributed/mpi_process_group.hpp", "libs/graph_parallel"),
238 std::pair<fs::path, fs::path>("boost/filesystem/config.hpp", "libs/filesystem/build"),
239 };
240
241 for(unsigned int n = 0; n < (sizeof(specials)/sizeof(specials[0])); ++n)
242 {
243 if(0 == compare_paths(specials[n].first, p))
244 {
245 if(!m_dependencies.count(specials[n].second))
246 {
247 m_dependencies[specials[n].second] = p; // set up dependency tree
248 add_path(specials[n].second);
249 }
250 }
251 }
252
253}
254
255void bcp_implementation::add_file_dependencies(const fs::path& p, bool scanfile)
256{
257 static const boost::regex e(
258 "^[[:blank:]]*(?://@bcp[[:blank:]]+([^\\n]*)\n)?#[[:blank:]]*include[[:blank:]]*[\"<]([^\">]+)[\">]"
259 );
260
261 if(!m_dependencies.count(p))
262 m_dependencies[p] = p; // set terminal dependency
263
264 fileview view;
265 if(scanfile)
266 view.open(p);
267 else
268 view.open(m_boost_path / p);
269 if(m_license_mode && !scanfile)
270 scan_license(p, view);
271 const int subs[] = { 1, 2 };
272 boost::regex_token_iterator<const char*> i(view.begin(), view.end(), e, subs);
273 boost::regex_token_iterator<const char*> j;
274 while(i != j)
275 {
276 //
277 // *i contains the name of the include file,
278 // check first for a file of that name in the
279 // same directory as the file we are scanning,
280 // and if that fails, then check the boost-root:
281 //
282 fs::path include_file;
283 try{
284 std::string discart_message = *i;
285 ++i;
286 if(discart_message.size())
287 {
288 // The include is optional and should be discarded:
289 std::cout << "Optional functionality won't be copied: " << discart_message << std::endl;
290 std::cout << "Add the file " << *i << " to the list of dependencies to extract to copy this functionality." << std::endl;
291 ++i;
292 continue;
293 }
294 include_file = i->str();
295 fs::path test_file(m_boost_path / p.branch_path() / include_file);
296 if(fs::exists(test_file) && !fs::is_directory(test_file) && (p.branch_path().string() != "boost"))
297 {
298 if(!m_dependencies.count(p.branch_path() / include_file))
299 {
300 m_dependencies[p.branch_path() / include_file] = p;
301 add_path(p.branch_path() / include_file);
302 }
303 }
304 else if(fs::exists(m_boost_path / include_file))
305 {
306 if(!m_dependencies.count(include_file))
307 {
308 m_dependencies[include_file] = p;
309 add_path(include_file);
310 }
311 }
312 ++i;
313 }
314 catch(const fs::filesystem_error&)
315 {
316 std::cerr << "Can't parse filename " << *i << " included by file " << p.string() << std::endl;
317 ++i;
318 continue;
319 }
320 }
321 //
322 // Now we need to scan for Boost.Preprocessor includes that
323 // are included via preprocessor iteration:
324 //
325 static const boost::regex ppfiles("^[[:blank:]]*#[[:blank:]]*define[[:blank:]]+(?:BOOST_PP_FILENAME|BOOST_PP_ITERATION_PARAMS|BOOST_PP_INDIRECT_SELF)[^\\n]+?[\"<]([^\">]+)[\">]");
326 i = boost::regex_token_iterator<const char*>(view.begin(), view.end(), ppfiles, 1);
327 while(i != j)
328 {
329 //
330 // *i contains the name of the include file,
331 // check first for a file of that name in the
332 // same directory as the file we are scanning,
333 // and if that fails, then check the boost-root:
334 //
335 fs::path include_file;
336 try{
337 include_file = i->str();
338 }
339 catch(const fs::filesystem_error&)
340 {
341 std::cerr << "Can't parse filename " << *i << " included by file " << p.string() << std::endl;
342 ++i;
343 continue;
344 }
345 fs::path test_file(m_boost_path / p.branch_path() / include_file);
346 if(fs::exists(test_file) && !fs::is_directory(test_file) && (p.branch_path().string() != "boost"))
347 {
348 if(!m_dependencies.count(p.branch_path() / include_file))
349 {
350 m_dependencies[p.branch_path() / include_file] = p;
351 add_path(p.branch_path() / include_file);
352 }
353 }
354 else if(fs::exists(m_boost_path / include_file))
355 {
356 if(!m_dependencies.count(include_file))
357 {
358 m_dependencies[include_file] = p;
359 add_path(include_file);
360 }
361 }
362 else
363 {
364 std::cerr << "CAUTION: Boost.Preprocessor iterated file " << include_file.string() << " does not exist." << std::endl;
365 }
366 ++i;
367 }
368
369 //
370 // Scan for any #include MACRO includes that we don't recognise.
371 //
372 // Begin by declaring all of the macros that get #included that
373 // we know about and are correctly handled as special cases:
374 //
375 static const std::string known_macros[] = {
376 "AUX778076_INCLUDE_STRING",
377 "BOOST_PP_STRINGIZE(boost/mpl/aux_/preprocessed/AUX778076_PREPROCESSED_HEADER)",
378 "BOOST_USER_CONFIG",
379 "BOOST_COMPILER_CONFIG",
380 "BOOST_STDLIB_CONFIG",
381 "BOOST_PLATFORM_CONFIG",
382 "BOOST_PP_FILENAME_1",
383 "BOOST_PP_ITERATION_PARAMS_1",
384 "BOOST_PP_FILENAME_2",
385 "BOOST_PP_ITERATION_PARAMS_2",
386 "BOOST_PP_FILENAME_3",
387 "BOOST_PP_ITERATION_PARAMS_3",
388 "BOOST_PP_FILENAME_4",
389 "BOOST_PP_ITERATION_PARAMS_4",
390 "BOOST_PP_FILENAME_5",
391 "BOOST_PP_ITERATION_PARAMS_5",
392 "BOOST_PP_INDIRECT_SELF",
393 "BOOST_PP_INCLUDE_SELF()",
394 "BOOST_PP_ITERATE",
395 "BOOST_PP_LOCAL_ITERATE",
396 "BOOST_PP_ITERATE()",
397 "BOOST_PP_LOCAL_ITERATE()",
398 "BOOST_PP_ASSIGN_SLOT(1)",
399 "BOOST_PP_ASSIGN_SLOT(2)",
400 "BOOST_PP_ASSIGN_SLOT(3)",
401 "BOOST_PP_ASSIGN_SLOT(4)",
402 "BOOST_PP_ASSIGN_SLOT(5)",
403 "BOOST_ABI_PREFIX",
404 "BOOST_ABI_SUFFIX",
405 "BOOST_PP_STRINGIZE(boost/mpl/aux_/preprocessed/AUX_PREPROCESSED_HEADER)",
406 "BOOST_PP_STRINGIZE(boost/mpl/list/AUX778076_HEADER)",
407 "BOOST_PP_STRINGIZE(boost/mpl/list/AUX778076_LIST_C_HEADER)",
408 "BOOST_PP_STRINGIZE(boost/mpl/list/AUX778076_LIST_HEADER)",
409 "BOOST_PP_STRINGIZE(boost/mpl/map/aux_/preprocessed/AUX778076_HEADER)",
410 "BOOST_PP_STRINGIZE(boost/mpl/map/AUX778076_MAP_HEADER)",
411 "BOOST_PP_STRINGIZE(boost/mpl/set/aux_/preprocessed/AUX778076_HEADER)",
412 "BOOST_PP_STRINGIZE(boost/mpl/set/AUX778076_SET_HEADER)",
413 "BOOST_PP_STRINGIZE(boost/mpl/set/AUX778076_SET_C_HEADER)",
414 "BOOST_PP_STRINGIZE(boost/mpl/vector/AUX778076_VECTOR_HEADER)",
415 "BOOST_PP_STRINGIZE(boost/mpl/vector/aux_/preprocessed/AUX778076_HEADER)",
416 "BOOST_PP_STRINGIZE(boost/mpl/vector/AUX778076_DEQUE_HEADER)",
417 "BOOST_PP_STRINGIZE(boost/mpl/vector/AUX778076_VECTOR_C_HEADER)",
418 "BOOST_REGEX_USER_CONFIG",
419 "BGL_PYTHON_EVENTS_HEADER",
420 "B1",
421 "B2",
422 "BOOST_TYPEOF_INCREMENT_REGISTRATION_GROUP()",
423 "BOOST_SLIST_HEADER",
424 "BOOST_HASH_SET_HEADER",
425 "BOOST_HASH_MAP_HEADER",
426 "BOOST_INTRUSIVE_INVARIANT_ASSERT_INCLUDE",
427 "BOOST_INTRUSIVE_SAFE_HOOK_DEFAULT_ASSERT_INCLUDE",
428 "BOOST_INTRUSIVE_SAFE_HOOK_DESTRUCTOR_ASSERT_INCLUDE",
429 "BOOST_FT_loop",
430 "BOOST_FT_AL_PREPROCESSED",
431 "BOOST_FT_AL_INCLUDE_FILE",
432 "__FILE__",
433 "BOOST_FT_cc_file",
434 "BOOST_FT_variate_file",
435 "BOOST_USER_CONFIG",
436 "BOOST_HEADER()",
437 "BOOST_TR1_STD_HEADER(utility)",
438 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(tuple))",
439 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(random))",
440 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(array))",
441 "BOOST_TR1_HEADER(cmath)",
442 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(complex))",
443 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(functional))",
444 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(memory))",
445 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(regex))",
446 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(type_traits))",
447 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(unordered_map))",
448 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(unordered_set))",
449 "BOOST_TR1_STD_HEADER(BOOST_TR1_PATH(utility))",
450 "BOOST_PROTO_LOCAL_ITERATE()",
451 "BOOST_SIGNAL_FUNCTION_N_HEADER",
452 "BOOST_PP_UPDATE_COUNTER()",
453 };
454
455 static const boost::regex indirect_includes("^[[:blank:]]*#[[:blank:]]*include[[:blank:]]+([^\"<][^\n]*?)[[:blank:]]*$");
456 i = boost::regex_token_iterator<const char*>(view.begin(), view.end(), indirect_includes, 1);
457 while(i != j)
458 {
459 const std::string* known_macros_end = known_macros + sizeof(known_macros)/sizeof(known_macros[0]);
460 if(known_macros_end == std::find(known_macros, known_macros_end, i->str()))
461 {
462 std::cerr << "CAUTION: don't know how to trace depenencies through macro: \"" << *i << "\" in file: " << p.string() << std::endl;
463 }
464 ++i;
465 }
466 //
467 // if the file contains a cpp_main / unit_test_main / test_main
468 // it is dependent upon Boost.test even if it doesn't
469 // include any of the Boost.test headers directly.
470 //
471 static const boost::regex m("^\\s*int\\s+(?:cpp_main|test_main|unit_test_main)");
472 boost::cmatch what;
473 if(boost::regex_search(view.begin(), view.end(), what, m))
474 {
475 add_dependent_lib("test", p, view);
476 }
477 if(!scanfile)
478 {
479 //
480 // grab the name of the library to which the header belongs,
481 // and if that library has source then add the source to our
482 // list:
483 //
484 // this regex catches boost/libname.hpp or boost/libname/whatever:
485 //
486 static const boost::regex lib1("boost/([^\\./]+)(?!detail).*");
487 boost::smatch swhat;
488 std::string gs(p.generic_string());
489 if(boost::regex_match(gs, swhat, lib1))
490 {
491 add_dependent_lib(swhat.str(1), p, view);
492 }
493 //
494 // and this one catches boost/x/y/whatever (for example numeric/ublas):
495 //
496 static const boost::regex lib2("boost/([^/]+/[^/]+)/(?!detail).*");
497 gs = p.generic_string();
498 if(boost::regex_match(gs, swhat, lib2))
499 {
500 add_dependent_lib(swhat.str(1), p, view);
501 }
502 }
503 if(m_list_namespaces)
504 {
505 //
506 // scan for top level namespaces and add to our list:
507 //
508 static const boost::regex namespace_scanner(
509 "^(?<!\\\\\\n)[[:blank:]]*+namespace\\s++(\\w++)\\s++(\\{[^{}]*(?:(?2)[^{}]*)*\\})"
510 );
511 i = boost::regex_token_iterator<const char*>(view.begin(), view.end(), namespace_scanner, 1);
512 while(i != j)
513 {
514 if(m_top_namespaces.count(*i) == 0)
515 {
516 //std::cout << *i << " (Found in " << p << ")" << std::endl;
517 m_top_namespaces[*i] = p;
518 }
519 ++i;
520 }
521 }
522}