Hide all uses of boost filesystem in fs.cpp. Use readdir as an

alternate implementation for Unix and add some feature checks so a
boost-free build of the tests and command line are possible again.

17 files changed, 157 insertions, 94 deletions

diff --git a/configure.py b/configure.py
index 0279361d8..aa863f639 100755
--- a/configure.py
+++ b/configure.py
@@ -259,6 +259,9 @@ def process_command_line(args):
                             dest='unaligned_mem', action='store_false',
                             help=optparse.SUPPRESS_HELP)
 
+    target_group.add_option('--with-os-features', action='append', help=optparse.SUPPRESS_HELP)
+    target_group.add_option('--without-os-features', action='append', help=optparse.SUPPRESS_HELP)
+
     for isa_extn_name in ['SSE2', 'SSSE3', 'AVX2', 'AES-NI', 'AltiVec']:
         isa_extn = isa_extn_name.lower()
 
@@ -459,6 +462,9 @@ def process_command_line(args):
     options.enabled_modules = parse_multiple_enable(options.enabled_modules)
     options.disabled_modules = parse_multiple_enable(options.disabled_modules)
 
+    options.with_os_features = parse_multiple_enable(options.with_os_features)
+    options.without_os_features = parse_multiple_enable(options.without_os_features)
+
     options.disable_intrinsics = parse_multiple_enable(options.disable_intrinsics)
 
     return options
@@ -948,10 +954,15 @@ class OsInfo(object):
     def ranlib_command(self):
         return ('ranlib' if self.ar_needs_ranlib else 'true')
 
-    def defines(self):
-        return ['TARGET_OS_IS_%s' % (self.basename.upper())] + \
-               ['TARGET_OS_HAS_' + feat.upper()
-                for feat in sorted(self.target_features)]
+    def defines(self, options):
+        r = []
+        for feat in self.target_features:
+            if feat not in options.without_os_features:
+                r += ['TARGET_OS_HAS_' + feat.upper()]
+        for feat in options.with_os_features:
+            if feat not in self.target_features:
+                r += ['TARGET_OS_HAS_' + feat.upper()]
+        return ['TARGET_OS_IS_%s' % (self.basename.upper())] + sorted(r)
 
 def fixup_proc_name(proc):
     proc = proc.lower().replace(' ', '')
@@ -1264,7 +1275,7 @@ def create_template_vars(build_config, options, modules, cc, arch, osinfo):
 
         'module_defines': make_cpp_macros(sorted(flatten([m.defines() for m in modules]))),
 
-        'target_os_defines': make_cpp_macros(osinfo.defines()),
+        'target_os_defines': make_cpp_macros(osinfo.defines(options)),
 
         'target_compiler_defines': make_cpp_macros(cc.defines()),
 
diff --git a/src/build-data/makefile/header.in b/src/build-data/makefile/header.in
index 6594cab69..d83529d5f 100644
--- a/src/build-data/makefile/header.in
+++ b/src/build-data/makefile/header.in
@@ -9,8 +9,8 @@ SO_OBJ_FLAGS   = %{shared_flags}
 LIB_LINK_CMD   = %{so_link}
 
 LIB_LINKS_TO   = %{link_to}
-APP_LINKS_TO   = $(LIB_LINKS_TO) -lboost_system
-TEST_LINKS_TO  = $(LIB_LINKS_TO) -lboost_filesystem -lboost_system
+APP_LINKS_TO   = $(LIB_LINKS_TO)
+TEST_LINKS_TO  = $(LIB_LINKS_TO)
 
 LIB_FLAGS      = $(SO_OBJ_FLAGS) $(LANG_FLAGS) $(LIB_OPT) $(WARN_FLAGS)
 APP_FLAGS      = $(LANG_FLAGS) $(APP_OPT) $(WARN_FLAGS)
diff --git a/src/build-data/os/darwin.txt b/src/build-data/os/darwin.txt
index d22b78b91..22e635352 100644
--- a/src/build-data/os/darwin.txt
+++ b/src/build-data/os/darwin.txt
@@ -13,6 +13,7 @@ dlopen
 gettimeofday
 gmtime_r
 memset_s
+readdir
 </target_features>
 
 <aliases>
diff --git a/src/build-data/os/freebsd.txt b/src/build-data/os/freebsd.txt
index 14e6c63e2..5df133c8e 100644
--- a/src/build-data/os/freebsd.txt
+++ b/src/build-data/os/freebsd.txt
@@ -1,6 +1,10 @@
 os_type unix
 
 <target_features>
-posix_mlock
+clock_gettime
 gettimeofday
+posix_mlock
+gmtime_r
+dlopen
+readdir
 </target_features>
diff --git a/src/build-data/os/linux.txt b/src/build-data/os/linux.txt
index 394f7673d..9061b31d8 100644
--- a/src/build-data/os/linux.txt
+++ b/src/build-data/os/linux.txt
@@ -6,6 +6,7 @@ gettimeofday
 posix_mlock
 gmtime_r
 dlopen
+readdir
 </target_features>
 
 <aliases>
diff --git a/src/build-data/os/netbsd.txt b/src/build-data/os/netbsd.txt
index 369b720c4..5df133c8e 100644
--- a/src/build-data/os/netbsd.txt
+++ b/src/build-data/os/netbsd.txt
@@ -1,5 +1,10 @@
 os_type unix
 
 <target_features>
+clock_gettime
 gettimeofday
+posix_mlock
+gmtime_r
+dlopen
+readdir
 </target_features>
diff --git a/src/build-data/os/openbsd.txt b/src/build-data/os/openbsd.txt
index 369b720c4..5df133c8e 100644
--- a/src/build-data/os/openbsd.txt
+++ b/src/build-data/os/openbsd.txt
@@ -1,5 +1,10 @@
 os_type unix
 
 <target_features>
+clock_gettime
 gettimeofday
+posix_mlock
+gmtime_r
+dlopen
+readdir
 </target_features>
diff --git a/src/cmd/credentials.h b/src/cmd/credentials.h
index b5ee701f3..e578b67a7 100644
--- a/src/cmd/credentials.h
+++ b/src/cmd/credentials.h
@@ -32,7 +32,7 @@ class Basic_Credentials_Manager : public Credentials_Manager
    public:
       Basic_Credentials_Manager()
          {
-
+         load_certstores();
          }
 
       Basic_Credentials_Manager(RandomNumberGenerator& rng,
diff --git a/src/cmd/tls_proxy.cpp b/src/cmd/tls_proxy.cpp
index d162ec0aa..7afe27a3c 100644
--- a/src/cmd/tls_proxy.cpp
+++ b/src/cmd/tls_proxy.cpp
@@ -7,7 +7,7 @@
 
 #include "apps.h"
 
-#if defined(BOTAN_HAS_TLS)
+#if defined(BOTAN_HAS_TLS) && defined(BOTAN_HAS_BOOST_ASIO)
 
 #include <iostream>
 #include <string>
diff --git a/src/lib/cert/x509/certstor.cpp b/src/lib/cert/x509/certstor.cpp
index cbb0fd670..d7e6f0a65 100644
--- a/src/lib/cert/x509/certstor.cpp
+++ b/src/lib/cert/x509/certstor.cpp
@@ -6,10 +6,7 @@
 */
 
 #include <botan/certstor.h>
-
-#if defined(BOTAN_HAS_BOOST_FILESYSTEM)
-#include <boost/filesystem.hpp>
-#endif
+#include <botan/fs.h>
 
 namespace Botan {
 
@@ -116,25 +113,17 @@ Certificate_Store_In_Memory::Certificate_Store_In_Memory(const std::string& dir)
    if(dir == "")
       return;
 
-#if defined(BOTAN_HAS_BOOST_FILESYSTEM)
-   boost::filesystem::recursive_directory_iterator i(dir);
-   boost::filesystem::recursive_directory_iterator end;
-
-   while(i != end)
+   std::vector<std::string> maybe_certs = list_all_readable_files_in_or_under(dir);
+   for(auto&& cert_file : maybe_certs)
       {
-      auto path = i->path();
-      ++i;
-
       try
          {
-         if(boost::filesystem::is_regular_file(path))
-            m_certs.push_back(X509_Certificate(path.string()));
+         m_certs.push_back(X509_Certificate(cert_file));
+         }
+      catch(std::exception&)
+         {
          }
-      catch(...) {}
       }
-#else
-   throw std::runtime_error("Certificate_Store_In_Memory: FS access disabled");
-#endif
    }
 
 const X509_Certificate*
diff --git a/src/lib/cert/x509/info.txt b/src/lib/cert/x509/info.txt
index 39e51a625..19db981bc 100644
--- a/src/lib/cert/x509/info.txt
+++ b/src/lib/cert/x509/info.txt
@@ -6,7 +6,3 @@ asn1
 datastor
 http_util
 </requires>
-
-<libs>
-all -> boost_filesystem
-</libs>
diff --git a/src/lib/entropy/proc_walk/proc_walk.cpp b/src/lib/entropy/proc_walk/proc_walk.cpp
index 616c76ea3..55affda40 100644
--- a/src/lib/entropy/proc_walk/proc_walk.cpp
+++ b/src/lib/entropy/proc_walk/proc_walk.cpp
@@ -43,11 +43,6 @@ class Directory_Walker : public File_Descriptor_Source
 
       int next_fd();
    private:
-      void add_directory(const std::string& dirname)
-         {
-         m_dirlist.push_back(dirname);
-         }
-
       std::pair<struct dirent*, std::string> get_next_dirent();
 
       std::pair<DIR*, std::string> m_cur_dir;
@@ -99,7 +94,7 @@ int Directory_Walker::next_fd()
 
       if(S_ISDIR(stat_buf.st_mode))
          {
-         add_directory(full_path);
+         m_dirlist.push_back(full_path);
          }
       else if(S_ISREG(stat_buf.st_mode) && (stat_buf.st_mode & S_IROTH))
          {
diff --git a/src/lib/utils/fs.cpp b/src/lib/utils/fs.cpp
new file mode 100644
index 000000000..268512325
--- /dev/null
+++ b/src/lib/utils/fs.cpp
@@ -0,0 +1,80 @@
+/*
+* (C) 2015 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/fs.h>
+#include <algorithm>
+#include <deque>
+
+#if defined(BOTAN_HAS_BOOST_FILESYSTEM)
+  #include <boost/filesystem.hpp>
+
+#elif defined(BOTAN_TARGET_OS_HAS_READDIR)
+  #include <sys/types.h>
+  #include <sys/stat.h>
+  #include <dirent.h>
+#endif
+
+namespace Botan {
+
+std::vector<std::string>
+list_all_readable_files_in_or_under(const std::string& dir_path)
+   {
+   std::vector<std::string> paths;
+
+#if defined(BOTAN_HAS_BOOST_FILESYSTEM)
+   namespace fs = boost::filesystem;
+
+   fs::recursive_directory_iterator end;
+   for(fs::recursive_directory_iterator dir(dir_path); dir != end; ++dir)
+      {
+      if(fs::is_regular_file(dir->path()))
+         paths.push_back(dir->path().string());
+      }
+
+#elif defined(BOTAN_TARGET_OS_HAS_READDIR)
+
+   std::deque<std::string> dir_list;
+   dir_list.push_back(dir_path);
+
+   while(!dir_list.empty())
+      {
+      const std::string cur_path = dir_list[0];
+      dir_list.pop_front();
+
+      std::unique_ptr<DIR, std::function<int (DIR*)>> dir(::opendir(cur_path.c_str()), ::closedir);
+
+      if(dir)
+         {
+         while(struct dirent* dirent = ::readdir(dir.get()))
+            {
+            const std::string filename = dirent->d_name;
+            if(filename == "." || filename == "..")
+               continue;
+            const std::string full_path = cur_path + '/' + filename;
+
+            struct stat stat_buf;
+
+            if(::lstat(full_path.c_str(), &stat_buf) == -1)
+               continue;
+
+            if(S_ISDIR(stat_buf.st_mode))
+               dir_list.push_back(full_path);
+            else if(S_ISREG(stat_buf.st_mode))
+               paths.push_back(full_path);
+            }
+         }
+      }
+#else
+  #warning "No filesystem access enabled"
+#endif
+
+   std::sort(paths.begin(), paths.end());
+
+   return paths;
+   }
+
+}
+
diff --git a/src/lib/utils/fs.h b/src/lib/utils/fs.h
new file mode 100644
index 000000000..25ec5ecd1
--- /dev/null
+++ b/src/lib/utils/fs.h
@@ -0,0 +1,21 @@
+/*
+* (C) 2015 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_UTIL_FS_H__
+#define BOTAN_UTIL_FS_H__
+
+#include <botan/types.h>
+#include <vector>
+#include <string>
+
+namespace Botan {
+
+BOTAN_DLL std::vector<std::string>
+list_all_readable_files_in_or_under(const std::string& dir);
+
+}
+
+#endif
diff --git a/src/lib/utils/info.txt b/src/lib/utils/info.txt
index fafb73c72..90f45aa3f 100644
--- a/src/lib/utils/info.txt
+++ b/src/lib/utils/info.txt
@@ -21,6 +21,7 @@ charset.h
 cpuid.h
 database.h
 exceptn.h
+fs.h
 get_byte.h
 loadstor.h
 mem_ops.h
diff --git a/src/tests/nist_x509.cpp b/src/tests/nist_x509.cpp
index 8c1f0d696..e6792512c 100644
--- a/src/tests/nist_x509.cpp
+++ b/src/tests/nist_x509.cpp
@@ -18,6 +18,7 @@
 #if defined(BOTAN_HAS_X509_CERTIFICATES)
 
 #include <botan/x509path.h>
+#include <botan/fs.h>
 
 #include <algorithm>
 #include <iostream>
@@ -26,38 +27,15 @@
 #include <vector>
 #include <map>
 #include <cstdlib>
-#include <boost/filesystem.hpp>
-
-namespace fs = boost::filesystem;
 
 using namespace Botan;
 
-namespace {
-
-std::vector<std::string> dir_listing(const std::string& dir_name)
-   {
-   std::vector<std::string> paths;
-
-   fs::directory_iterator dir(dir_name), end;
-
-   while(dir != end)
-      {
-      paths.push_back(dir->path().string());
-      ++dir;
-      }
-
-   std::sort(paths.begin(), paths.end());
-
-   return paths;
-   }
-
-}
-
 std::map<size_t, Path_Validation_Result::Code> get_expected();
 
 size_t test_nist_x509()
    {
    const std::string root_test_dir = "src/tests/data/nist_x509/";
+   const size_t total_tests = 76;
 
    size_t unexp_failure = 0;
    size_t unexp_success = 0;
@@ -69,14 +47,10 @@ size_t test_nist_x509()
 
    try {
 
-   const std::vector<std::string> test_dirs = dir_listing(root_test_dir);
-
-   for(size_t i = 0; i != test_dirs.size(); i++)
+   for(size_t test_no = 1; test_no <= total_tests; ++test_no)
       {
-      const size_t test_no = i+1;
-
-      const std::string test_dir = test_dirs[i];
-      const std::vector<std::string> all_files = dir_listing(test_dir);
+      const std::string test_dir = root_test_dir + "/test" + (test_no <= 9 ? "0" : "") + std::to_string(test_no);
+      const std::vector<std::string> all_files = list_all_readable_files_in_or_under(test_dir);
 
       std::vector<std::string> certs, crls;
       std::string root_cert, to_verify;
@@ -96,7 +70,7 @@ size_t test_nist_x509()
             crls.push_back(current);
          }
 
-      if(expected_results.find(i+1) == expected_results.end())
+      if(expected_results.find(test_no) == expected_results.end())
          {
          skipped++;
          continue;
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 9a0a29bdd..299d07403 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -8,47 +8,27 @@
 #include <iostream>
 #include <fstream>
 #include <botan/auto_rng.h>
+#include <botan/fs.h>
 
 #if defined(BOTAN_HAS_SYSTEM_RNG)
   #include <botan/system_rng.h>
 #endif
 
-#include <boost/filesystem.hpp>
-
-namespace fs = boost::filesystem;
-
 Botan::RandomNumberGenerator& test_rng()
    {
 #if defined(BOTAN_HAS_SYSTEM_RNG)
    return Botan::system_rng();
 #else
-   static AutoSeeded_RNG rng;
+   static Botan::AutoSeeded_RNG rng;
    return rng;
 #endif
    }
 
-std::vector<std::string> list_dir(const std::string& dir_path)
-   {
-   std::vector<std::string> paths;
-
-   fs::recursive_directory_iterator dir(dir_path), end;
-
-   while (dir != end)
-      {
-      if(dir->path().extension().string() == ".vec")
-         paths.push_back(dir->path().string());
-      ++dir;
-      }
-
-   std::sort(paths.begin(), paths.end());
-
-   return paths;
-   }
-
 size_t run_tests_in_dir(const std::string& dir, std::function<size_t (const std::string&)> fn)
    {
    size_t fails = 0;
-   for(auto vec: list_dir(dir))
+
+   for(auto vec: Botan::list_all_readable_files_in_or_under(dir))
       fails += fn(vec);
    return fails;
    }