Refactor option parsing in cli and test code

Allows cleaning up header includes, also somewhat smaller binaries.

15 files changed, 730 insertions, 536 deletions

diff --git a/src/cli/argparse.h b/src/cli/argparse.h
new file mode 100644
index 000000000..457716095
--- /dev/null
+++ b/src/cli/argparse.h
@@ -0,0 +1,278 @@
+/*
+* (C) 2015,2017 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_ARGPARSE_H_
+#define BOTAN_ARGPARSE_H_
+
+#include <string>
+#include <map>
+#include <set>
+#include <vector>
+#include <botan/parsing.h>
+#include "cli_exceptions.h"
+
+namespace Botan_CLI {
+
+class Argument_Parser
+   {
+   public:
+      Argument_Parser(const std::string& spec,
+                      const std::vector<std::string>& extra_flags = {},
+                      const std::vector<std::string>& extra_opts = {});
+
+      void parse_args(const std::vector<std::string>& params);
+
+      bool flag_set(const std::string& flag) const;
+
+      bool has_arg(const std::string& opt_name) const;
+      std::string get_arg(const std::string& option) const;
+
+      std::string get_arg_or(const std::string& option, const std::string& otherwise) const;
+
+      size_t get_arg_sz(const std::string& option) const;
+
+      std::vector<std::string> get_arg_list(const std::string& what) const;
+
+   private:
+      // set in constructor
+      std::vector<std::string> m_spec_args;
+      std::set<std::string> m_spec_flags;
+      std::map<std::string, std::string> m_spec_opts;
+      std::string m_spec_rest;
+
+      // set in parse_args()
+      std::map<std::string, std::string> m_user_args;
+      std::set<std::string> m_user_flags;
+      std::vector<std::string> m_user_rest;
+   };
+
+bool Argument_Parser::flag_set(const std::string& flag_name) const
+   {
+   return m_user_flags.count(flag_name) > 0;
+   }
+
+bool Argument_Parser::has_arg(const std::string& opt_name) const
+   {
+   return m_user_args.count(opt_name) > 0;
+   }
+
+std::string Argument_Parser::get_arg(const std::string& opt_name) const
+   {
+   auto i = m_user_args.find(opt_name);
+   if(i == m_user_args.end())
+      {
+      // this shouldn't occur unless you passed the wrong thing to get_arg
+      throw CLI_Error("Unknown option " + opt_name + " used (program bug)");
+      }
+   return i->second;
+   }
+
+std::string Argument_Parser::get_arg_or(const std::string& opt_name, const std::string& otherwise) const
+   {
+   auto i = m_user_args.find(opt_name);
+   if(i == m_user_args.end() || i->second.empty())
+      {
+      return otherwise;
+      }
+   return i->second;
+   }
+
+size_t Argument_Parser::get_arg_sz(const std::string& opt_name) const
+   {
+   const std::string s = get_arg(opt_name);
+
+   try
+      {
+      return static_cast<size_t>(std::stoul(s));
+      }
+   catch(std::exception&)
+      {
+      throw CLI_Usage_Error("Invalid integer value '" + s + "' for option " + opt_name);
+      }
+   }
+
+std::vector<std::string> Argument_Parser::get_arg_list(const std::string& what) const
+   {
+   if(what != m_spec_rest)
+      {
+      throw CLI_Error("Unexpected list name '" + what + "'");
+      }
+   return m_user_rest;
+   }
+
+void Argument_Parser::parse_args(const std::vector<std::string>& params)
+   {
+   std::vector<std::string> args;
+   for(auto const& param : params)
+      {
+      if(param.find("--") == 0)
+         {
+         // option
+         const auto eq = param.find('=');
+
+         if(eq == std::string::npos)
+            {
+            const std::string opt_name = param.substr(2, std::string::npos);
+
+            if(m_spec_flags.count(opt_name) == 0)
+               {
+               if(m_spec_opts.count(opt_name))
+                  {
+                  throw CLI_Usage_Error("Invalid usage of option --" + opt_name +
+                                        " without value");
+                  }
+               else
+                  {
+                  throw CLI_Usage_Error("Unknown flag --" + opt_name);
+                  }
+               }
+            m_user_flags.insert(opt_name);
+            }
+         else
+            {
+            const std::string opt_name = param.substr(2, eq - 2);
+            const std::string opt_val = param.substr(eq + 1, std::string::npos);
+
+            if(m_spec_opts.count(opt_name) == 0)
+               {
+               throw CLI_Usage_Error("Unknown option --" + opt_name);
+               }
+
+            m_user_args.insert(std::make_pair(opt_name, opt_val));
+            }
+         }
+      else
+         {
+         // argument
+         args.push_back(param);
+         }
+      }
+
+   bool seen_stdin_flag = false;
+   size_t arg_i = 0;
+   for(auto const& arg : m_spec_args)
+      {
+      if(arg_i >= args.size())
+         {
+         // not enough arguments
+         throw CLI_Usage_Error("Invalid argument count, got " +
+                               std::to_string(args.size()) +
+                               " expected " +
+                               std::to_string(m_spec_args.size()));
+         }
+
+      m_user_args.insert(std::make_pair(arg, args[arg_i]));
+
+      if(args[arg_i] == "-")
+         {
+         if(seen_stdin_flag)
+            {
+            throw CLI_Usage_Error("Cannot specify '-' (stdin) more than once");
+            }
+         seen_stdin_flag = true;
+         }
+
+      ++arg_i;
+      }
+
+   if(m_spec_rest.empty())
+      {
+      if(arg_i != args.size())
+         {
+         throw CLI_Usage_Error("Too many arguments");
+         }
+      }
+   else
+      {
+      m_user_rest.assign(args.begin() + arg_i, args.end());
+      }
+
+   // Now insert any defaults for options not supplied by the user
+   for(auto const& opt : m_spec_opts)
+      {
+      if(m_user_args.count(opt.first) == 0)
+         {
+         m_user_args.insert(opt);
+         }
+      }
+   }
+
+Argument_Parser::Argument_Parser(const std::string& spec,
+                                 const std::vector<std::string>& extra_flags,
+                                 const std::vector<std::string>& extra_opts)
+   {
+   class CLI_Error_Invalid_Spec : public CLI_Error
+      {
+      public:
+         explicit CLI_Error_Invalid_Spec(const std::string& spec)
+            : CLI_Error("Invalid command spec '" + spec + "'") {}
+      };
+
+   const std::vector<std::string> parts = Botan::split_on(spec, ' ');
+
+   if(parts.size() == 0)
+      {
+      throw CLI_Error_Invalid_Spec(spec);
+      }
+
+   for(size_t i = 1; i != parts.size(); ++i)
+      {
+      const std::string s = parts[i];
+
+      if(s.empty()) // ?!? (shouldn't happen)
+         {
+         throw CLI_Error_Invalid_Spec(spec);
+         }
+
+      if(s.size() > 2 && s[0] == '-' && s[1] == '-')
+         {
+         // option or flag
+
+         auto eq = s.find('=');
+
+         if(eq == std::string::npos)
+            {
+            m_spec_flags.insert(s.substr(2, std::string::npos));
+            }
+         else
+            {
+            m_spec_opts.insert(std::make_pair(s.substr(2, eq - 2), s.substr(eq + 1, std::string::npos)));
+            }
+         }
+      else if(s[0] == '*')
+         {
+         // rest argument
+         if(m_spec_rest.empty() && s.size() > 2)
+            {
+            m_spec_rest = s.substr(1, std::string::npos);
+            }
+         else
+            {
+            throw CLI_Error_Invalid_Spec(spec);
+            }
+         }
+      else
+         {
+         // named argument
+         if(!m_spec_rest.empty()) // rest arg wasn't last
+            {
+            throw CLI_Error_Invalid_Spec(spec);
+            }
+
+         m_spec_args.push_back(s);
+         }
+      }
+
+   for(std::string flag : extra_flags)
+      m_spec_flags.insert(flag);
+   for(std::string opt : extra_opts)
+      m_spec_opts.insert(std::make_pair(opt, ""));
+   }
+
+
+}
+
+#endif
diff --git a/src/cli/cli.cpp b/src/cli/cli.cpp
new file mode 100644
index 000000000..f23c3574d
--- /dev/null
+++ b/src/cli/cli.cpp
@@ -0,0 +1,250 @@
+/*
+* (C) 2015,2017 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include "cli.h"
+#include "argparse.h"
+#include <botan/rng.h>
+#include <botan/parsing.h>
+#include <iostream>
+#include <fstream>
+
+namespace Botan_CLI {
+
+Command::Command(const std::string& cmd_spec) : m_spec(cmd_spec)
+   {
+   // for checking all spec strings at load time
+   //m_args.reset(new Argument_Parser(m_spec));
+   }
+
+Command::~Command() { /* for unique_ptr */ }
+
+std::string Command::help_text() const
+   {
+   return "Usage: " + m_spec +
+      "\n\nAll commands support --verbose --help --output= --error-output= --rng-type= --drbg-seed=";
+   }
+
+int Command::run(const std::vector<std::string>& params)
+   {
+   try
+      {
+      m_args.reset(new Argument_Parser(m_spec,
+                                       {"verbose", "help"},
+                                       {"output", "error-output", "rng-type", "drbg-seed"}));
+
+      m_args->parse_args(params);
+
+      if(m_args->has_arg("output"))
+         {
+         const std::string output_file = get_arg("output");
+
+         if(output_file != "")
+            {
+            m_output_stream.reset(new std::ofstream(output_file, std::ios::binary));
+            if(!m_output_stream->good())
+               throw CLI_IO_Error("opening", output_file);
+            }
+         }
+
+      if(m_args->has_arg("error-output"))
+         {
+         const std::string output_file = get_arg("error-output");
+
+         if(output_file != "")
+            {
+            m_error_output_stream.reset(new std::ofstream(output_file, std::ios::binary));
+            if(!m_error_output_stream->good())
+               throw CLI_IO_Error("opening", output_file);
+            }
+         }
+
+      if(flag_set("help"))
+         {
+         output() << help_text() << "\n";
+         return 2;
+         }
+
+      this->go();
+      return 0;
+      }
+   catch(CLI_Usage_Error& e)
+      {
+      error_output() << "Usage error: " << e.what() << "\n";
+      error_output() << help_text() << "\n";
+      return 1;
+      }
+   catch(std::exception& e)
+      {
+      error_output() << "Error: " << e.what() << "\n";
+      return 2;
+      }
+   catch(...)
+      {
+      error_output() << "Error: unknown exception\n";
+      return 2;
+      }
+   }
+
+bool Command::flag_set(const std::string& flag_name) const
+   {
+   return m_args->flag_set(flag_name);
+   }
+
+std::string Command::get_arg(const std::string& opt_name) const
+   {
+   return m_args->get_arg(opt_name);
+   }
+
+/*
+* Like get_arg() but if the argument was not specified or is empty, returns otherwise
+*/
+std::string Command::get_arg_or(const std::string& opt_name, const std::string& otherwise) const
+   {
+   return m_args->get_arg_or(opt_name, otherwise);
+   }
+
+size_t Command::get_arg_sz(const std::string& opt_name) const
+   {
+   return m_args->get_arg_sz(opt_name);
+   }
+
+std::vector<std::string> Command::get_arg_list(const std::string& what) const
+   {
+   return m_args->get_arg_list(what);
+   }
+
+std::ostream& Command::output()
+   {
+   if(m_output_stream.get())
+      {
+      return *m_output_stream;
+      }
+   return std::cout;
+   }
+
+std::ostream& Command::error_output()
+   {
+   if(m_error_output_stream.get())
+      {
+      return *m_error_output_stream;
+      }
+   return std::cerr;
+   }
+
+std::vector<uint8_t> Command::slurp_file(const std::string& input_file,
+                                         size_t buf_size) const
+   {
+   std::vector<uint8_t> buf;
+   auto insert_fn = [&](const uint8_t b[], size_t l)
+      {
+      buf.insert(buf.end(), b, b + l);
+      };
+   this->read_file(input_file, insert_fn, buf_size);
+   return buf;
+   }
+
+std::string Command::slurp_file_as_str(const std::string& input_file,
+                                       size_t buf_size) const
+   {
+   std::string str;
+   auto insert_fn = [&](const uint8_t b[], size_t l)
+      {
+      str.append(reinterpret_cast<const char*>(b), l);
+      };
+   this->read_file(input_file, insert_fn, buf_size);
+   return str;
+   }
+
+void Command::read_file(const std::string& input_file,
+                        std::function<void (uint8_t[], size_t)> consumer_fn,
+                        size_t buf_size) const
+   {
+   if(input_file == "-")
+      {
+      do_read_file(std::cin, consumer_fn, buf_size);
+      }
+   else
+      {
+      std::ifstream in(input_file, std::ios::binary);
+      if(!in)
+         {
+         throw CLI_IO_Error("reading file", input_file);
+         }
+      do_read_file(in, consumer_fn, buf_size);
+      }
+   }
+
+void Command::do_read_file(std::istream& in,
+                  std::function<void (uint8_t[], size_t)> consumer_fn,
+                  size_t buf_size) const
+   {
+   // Avoid an infinite loop on --buf-size=0
+   std::vector<uint8_t> buf(buf_size == 0 ? 4096 : buf_size);
+
+   while(in.good())
+      {
+      in.read(reinterpret_cast<char*>(buf.data()), buf.size());
+      const size_t got = static_cast<size_t>(in.gcount());
+      consumer_fn(buf.data(), got);
+      }
+   }
+
+Botan::RandomNumberGenerator& Command::rng()
+   {
+   if(m_rng == nullptr)
+      {
+      m_rng = cli_make_rng(get_arg("rng-type"), get_arg("drbg-seed"));
+      }
+
+   return *m_rng.get();
+   }
+
+// Registration code
+
+Command::Registration::Registration(const std::string& name, Command::cmd_maker_fn maker_fn)
+   {
+   std::map<std::string, Command::cmd_maker_fn>& reg = Command::global_registry();
+
+   if(reg.count(name) > 0)
+      {
+      throw CLI_Error("Duplicated registration of command " + name);
+      }
+
+   reg.insert(std::make_pair(name, maker_fn));
+   }
+
+//static
+std::map<std::string, Command::cmd_maker_fn>& Command::global_registry()
+   {
+   static std::map<std::string, Command::cmd_maker_fn> g_cmds;
+   return g_cmds;
+   }
+
+//static
+std::vector<std::string> Command::registered_cmds()
+   {
+   std::vector<std::string> cmds;
+   for(auto& cmd : Command::global_registry())
+      cmds.push_back(cmd.first);
+   return cmds;
+   }
+
+//static
+std::unique_ptr<Command> Command::get_cmd(const std::string& name)
+   {
+   const std::map<std::string, Command::cmd_maker_fn>& reg = Command::global_registry();
+
+   std::unique_ptr<Command> r;
+   auto i = reg.find(name);
+   if(i != reg.end())
+      {
+      r.reset(i->second());
+      }
+
+   return r;
+   }
+
+}
diff --git a/src/cli/cli.h b/src/cli/cli.h
index 1e81c2ff6..75de33515 100644
--- a/src/cli/cli.h
+++ b/src/cli/cli.h
@@ -8,64 +8,39 @@
 #define BOTAN_CLI_H_
 
 #include <botan/build.h>
-#include <botan/parsing.h>
-#include <botan/rng.h>
-
-#include <fstream>
-#include <iostream>
 #include <functional>
+#include <ostream>
 #include <map>
 #include <memory>
-#include <set>
 #include <string>
 #include <vector>
+#include "cli_exceptions.h"
+
+namespace Botan {
+
+class RandomNumberGenerator;
+
+}
 
 namespace Botan_CLI {
 
-/* Declared in utils.cpp */
+class Argument_Parser;
+
+/* Declared in cli_rng.cpp */
 std::unique_ptr<Botan::RandomNumberGenerator>
 cli_make_rng(const std::string& type, const std::string& hex_drbg_seed);
 
-class CLI_Error : public std::runtime_error
-   {
-   public:
-      explicit CLI_Error(const std::string& s) : std::runtime_error(s) {}
-   };
-
-class CLI_IO_Error : public CLI_Error
+class Command
    {
    public:
-      CLI_IO_Error(const std::string& op, const std::string& who) :
-         CLI_Error("Error " + op + " " + who) {}
-   };
 
-class CLI_Usage_Error : public CLI_Error
-   {
-   public:
-      explicit CLI_Usage_Error(const std::string& what) : CLI_Error(what) {}
-   };
+      /**
+      * Get a registered command
+      */
+      static std::unique_ptr<Command> get_cmd(const std::string& name);
 
-/* Thrown eg when a requested feature was compiled out of the library
-   or is not available, eg hashing with
-*/
-class CLI_Error_Unsupported : public CLI_Error
-   {
-   public:
-      CLI_Error_Unsupported(const std::string& what,
-                            const std::string& who)
-         : CLI_Error(what + " with '" + who + "' unsupported or not available") {}
-   };
+      static std::vector<std::string> registered_cmds();
 
-class CLI_Error_Invalid_Spec : public CLI_Error
-   {
-   public:
-      explicit CLI_Error_Invalid_Spec(const std::string& spec)
-         : CLI_Error("Invalid command spec '" + spec + "'") {}
-   };
-
-class Command
-   {
-   public:
       /**
       * The spec string specifies the format of the command line, eg for
       * a somewhat complicated command:
@@ -105,156 +80,13 @@ class Command
       * Use of --help is captured in run() and returns help_text().
       * Use of --verbose can be checked with verbose() or flag_set("verbose")
       */
-      explicit Command(const std::string& cmd_spec) : m_spec(cmd_spec)
-         {
-         // for checking all spec strings at load time
-         //parse_spec();
-         }
-      virtual ~Command() = default;
+      explicit Command(const std::string& cmd_spec);
 
-      int run(const std::vector<std::string>& params)
-         {
-         try
-            {
-            // avoid parsing specs except for the command actually running
-            parse_spec();
-
-            std::vector<std::string> args;
-            for(auto const& param : params)
-               {
-               if(param.find("--") == 0)
-                  {
-                  // option
-                  const auto eq = param.find('=');
-
-                  if(eq == std::string::npos)
-                     {
-                     const std::string opt_name = param.substr(2, std::string::npos);
-
-                     if(m_spec_flags.count(opt_name) == 0)
-                        {
-                        if(m_spec_opts.count(opt_name))
-                           {
-                           throw CLI_Usage_Error("Invalid usage of option --" + opt_name +
-                                                 " without value");
-                           }
-                        else
-                           {
-                           throw CLI_Usage_Error("Unknown flag --" + opt_name);
-                           }
-                        }
-                     m_user_flags.insert(opt_name);
-                     }
-                  else
-                     {
-                     const std::string opt_name = param.substr(2, eq - 2);
-                     const std::string opt_val = param.substr(eq + 1, std::string::npos);
-
-                     if(m_spec_opts.count(opt_name) == 0)
-                        {
-                        throw CLI_Usage_Error("Unknown option --" + opt_name);
-                        }
-
-                     m_user_args.insert(std::make_pair(opt_name, opt_val));
-                     }
-                  }
-               else
-                  {
-                  // argument
-                  args.push_back(param);
-                  }
-               }
-
-            bool seen_stdin_flag = false;
-            size_t arg_i = 0;
-            for(auto const& arg : m_spec_args)
-               {
-               if(arg_i >= args.size())
-                  {
-                  // not enough arguments
-                  throw CLI_Usage_Error("Invalid argument count, got " +
-                                        std::to_string(args.size()) +
-                                        " expected " +
-                                        std::to_string(m_spec_args.size()));
-                  }
-
-               m_user_args.insert(std::make_pair(arg, args[arg_i]));
-
-               if(args[arg_i] == "-")
-                  {
-                  if(seen_stdin_flag)
-                     {
-                     throw CLI_Usage_Error("Cannot specifiy '-' (stdin) more than once");
-                     }
-                  seen_stdin_flag = true;
-                  }
-
-               ++arg_i;
-               }
-
-            if(m_spec_rest.empty())
-               {
-               if(arg_i != args.size())
-                  {
-                  throw CLI_Usage_Error("Too many arguments");
-                  }
-               }
-            else
-               {
-               m_user_rest.assign(args.begin() + arg_i, args.end());
-               }
-
-            if(flag_set("help"))
-               {
-               output() << help_text() << "\n";
-               return 2;
-               }
-
-            if(m_user_args.count("output"))
-               {
-               m_output_stream.reset(new std::ofstream(get_arg("output"), std::ios::binary));
-               }
-
-            if(m_user_args.count("error-output"))
-               {
-               m_error_output_stream.reset(new std::ofstream(get_arg("error-output"), std::ios::binary));
-               }
-
-            // Now insert any defaults for options not supplied by the user
-            for(auto const& opt : m_spec_opts)
-               {
-               if(m_user_args.count(opt.first) == 0)
-                  {
-                  m_user_args.insert(opt);
-                  }
-               }
-
-            this->go();
-            return 0;
-            }
-         catch(CLI_Usage_Error& e)
-            {
-            error_output() << "Usage error: " << e.what() << "\n";
-            error_output() << help_text() << "\n";
-            return 1;
-            }
-         catch(std::exception& e)
-            {
-            error_output() << "Error: " << e.what() << "\n";
-            return 2;
-            }
-         catch(...)
-            {
-            error_output() << "Error: unknown exception\n";
-            return 2;
-            }
-         }
+      virtual ~Command();
 
-      virtual std::string help_text() const
-         {
-         return "Usage: " + m_spec +
-            "\n\nAll commands support --verbose --help --output= --error-output= --rng-type= --drbg-seed=";
-         }
+      int run(const std::vector<std::string>& params);
+
+      virtual std::string help_text() const;
 
       const std::string& cmd_spec() const
          {
@@ -268,230 +100,53 @@ class Command
 
    protected:
 
-      void parse_spec()
-         {
-         const std::vector<std::string> parts = Botan::split_on(m_spec, ' ');
-
-         if(parts.size() == 0)
-            {
-            throw CLI_Error_Invalid_Spec(m_spec);
-            }
-
-         for(size_t i = 1; i != parts.size(); ++i)
-            {
-            const std::string s = parts[i];
-
-            if(s.empty()) // ?!? (shouldn't happen)
-               {
-               throw CLI_Error_Invalid_Spec(m_spec);
-               }
-
-            if(s.size() > 2 && s[0] == '-' && s[1] == '-')
-               {
-               // option or flag
-
-               auto eq = s.find('=');
-
-               if(eq == std::string::npos)
-                  {
-                  m_spec_flags.insert(s.substr(2, std::string::npos));
-                  }
-               else
-                  {
-                  m_spec_opts.insert(std::make_pair(s.substr(2, eq - 2), s.substr(eq + 1, std::string::npos)));
-                  }
-               }
-            else if(s[0] == '*')
-               {
-               // rest argument
-               if(m_spec_rest.empty() && s.size() > 2)
-                  {
-                  m_spec_rest = s.substr(1, std::string::npos);
-                  }
-               else
-                  {
-                  throw CLI_Error_Invalid_Spec(m_spec);
-                  }
-               }
-            else
-               {
-               // named argument
-               if(!m_spec_rest.empty()) // rest arg wasn't last
-                  {
-                  throw CLI_Error("Invalid command spec " + m_spec);
-                  }
-
-               m_spec_args.push_back(s);
-               }
-            }
-
-         m_spec_flags.insert("verbose");
-         m_spec_flags.insert("help");
-         m_spec_opts.insert(std::make_pair("output", ""));
-         m_spec_opts.insert(std::make_pair("error-output", ""));
-         m_spec_opts.insert(std::make_pair("rng-type", ""));
-         m_spec_opts.insert(std::make_pair("drbg-seed", ""));
-         }
-
       /*
       * The actual functionality of the cli command implemented in subclas
       */
       virtual void go() = 0;
 
-      std::ostream& output()
-         {
-         if(m_output_stream.get())
-            {
-            return *m_output_stream;
-            }
-         return std::cout;
-         }
+      std::ostream& output();
 
-      std::ostream& error_output()
-         {
-         if(m_error_output_stream.get())
-            {
-            return *m_error_output_stream;
-            }
-         return std::cerr;
-         }
+      std::ostream& error_output();
 
       bool verbose() const
          {
          return flag_set("verbose");
          }
 
-      bool flag_set(const std::string& flag_name) const
-         {
-         return m_user_flags.count(flag_name) > 0;
-         }
+      bool flag_set(const std::string& flag_name) const;
 
-      std::string get_arg(const std::string& opt_name) const
-         {
-         auto i = m_user_args.find(opt_name);
-         if(i == m_user_args.end())
-            {
-            // this shouldn't occur unless you passed the wrong thing to get_arg
-            throw CLI_Error("Unknown option " + opt_name + " used (program bug)");
-            }
-         return i->second;
-         }
+      std::string get_arg(const std::string& opt_name) const;
 
       /*
       * Like get_arg() but if the argument was not specified or is empty, returns otherwise
       */
-      std::string get_arg_or(const std::string& opt_name, const std::string& otherwise) const
-         {
-         auto i = m_user_args.find(opt_name);
-         if(i == m_user_args.end() || i->second.empty())
-            {
-            return otherwise;
-            }
-         return i->second;
-         }
+      std::string get_arg_or(const std::string& opt_name, const std::string& otherwise) const;
 
-      size_t get_arg_sz(const std::string& opt_name) const
-         {
-         const std::string s = get_arg(opt_name);
-
-         try
-            {
-            return static_cast<size_t>(std::stoul(s));
-            }
-         catch(std::exception&)
-            {
-            throw CLI_Usage_Error("Invalid integer value '" + s + "' for option " + opt_name);
-            }
-         }
+      size_t get_arg_sz(const std::string& opt_name) const;
 
-      std::vector<std::string> get_arg_list(const std::string& what) const
-         {
-         if(what != m_spec_rest)
-            {
-            throw CLI_Error("Unexpected list name '" + what + "'");
-            }
-
-         return m_user_rest;
-         }
+      std::vector<std::string> get_arg_list(const std::string& what) const;
 
       /*
       * Read an entire file into memory and return the contents
       */
       std::vector<uint8_t> slurp_file(const std::string& input_file,
-                                      size_t buf_size = 0) const
-         {
-         std::vector<uint8_t> buf;
-         auto insert_fn = [&](const uint8_t b[], size_t l)
-            {
-            buf.insert(buf.end(), b, b + l);
-            };
-         this->read_file(input_file, insert_fn, buf_size);
-         return buf;
-         }
-
-      /*
-      * Read an entire file into memory and return the contents
-      */
-      Botan::secure_vector<uint8_t> slurp_file_locked(const std::string& input_file,
-                                                      size_t buf_size = 0) const
-         {
-         Botan::secure_vector<uint8_t> buf;
-         auto insert_fn = [&](const uint8_t b[], size_t l)
-            {
-            buf.insert(buf.end(), b, b + l);
-            };
-         this->read_file(input_file, insert_fn, buf_size);
-         return buf;
-         }
+                                      size_t buf_size = 0) const;
 
       std::string slurp_file_as_str(const std::string& input_file,
-                                    size_t buf_size = 0) const
-         {
-         std::string str;
-         auto insert_fn = [&](const uint8_t b[], size_t l)
-            {
-            str.append(reinterpret_cast<const char*>(b), l);
-            };
-         this->read_file(input_file, insert_fn, buf_size);
-         return str;
-         }
+                                    size_t buf_size = 0) const;
 
       /*
       * Read a file calling consumer_fn() with the inputs
       */
       void read_file(const std::string& input_file,
                      std::function<void (uint8_t[], size_t)> consumer_fn,
-                     size_t buf_size = 0) const
-         {
-         if(input_file == "-")
-            {
-            do_read_file(std::cin, consumer_fn, buf_size);
-            }
-         else
-            {
-            std::ifstream in(input_file, std::ios::binary);
-            if(!in)
-               {
-               throw CLI_IO_Error("reading file", input_file);
-               }
-            do_read_file(in, consumer_fn, buf_size);
-            }
-         }
+                     size_t buf_size = 0) const;
+
 
       void do_read_file(std::istream& in,
                         std::function<void (uint8_t[], size_t)> consumer_fn,
-                        size_t buf_size = 0) const
-         {
-         // Avoid an infinite loop on --buf-size=0
-         std::vector<uint8_t> buf(buf_size == 0 ? 4096 : buf_size);
-
-         while(in.good())
-            {
-            in.read(reinterpret_cast<char*>(buf.data()), buf.size());
-            const size_t got = static_cast<size_t>(in.gcount());
-            consumer_fn(buf.data(), got);
-            }
-         }
+                        size_t buf_size = 0) const;
 
       template<typename Alloc>
       void write_output(const std::vector<uint8_t, Alloc>& vec)
@@ -499,75 +154,31 @@ class Command
          output().write(reinterpret_cast<const char*>(vec.data()), vec.size());
          }
 
-      Botan::RandomNumberGenerator& rng()
-         {
-         if(m_rng == nullptr)
-            {
-            m_rng = cli_make_rng(get_arg("rng-type"), get_arg("drbg-seed"));
-            }
-
-         return *m_rng.get();
-         }
+      Botan::RandomNumberGenerator& rng();
 
    private:
-      // set in constructor
-      std::string m_spec;
+      void parse_spec();
 
-      // set in parse_spec() from m_spec
-      std::vector<std::string> m_spec_args;
-      std::set<std::string> m_spec_flags;
-      std::map<std::string, std::string> m_spec_opts;
-      std::string m_spec_rest;
+      typedef std::function<Command* ()> cmd_maker_fn;
 
-      // set in run() from user args
-      std::map<std::string, std::string> m_user_args;
-      std::set<std::string> m_user_flags;
-      std::vector<std::string> m_user_rest;
+      static std::map<std::string, cmd_maker_fn>& global_registry();
 
-      std::unique_ptr<std::ofstream> m_output_stream;
-      std::unique_ptr<std::ofstream> m_error_output_stream;
+      // set in constructor
+      std::string m_spec;
+
+      std::unique_ptr<Argument_Parser> m_args;
+      std::unique_ptr<std::ostream> m_output_stream;
+      std::unique_ptr<std::ostream> m_error_output_stream;
 
       std::unique_ptr<Botan::RandomNumberGenerator> m_rng;
 
    public:
       // the registry interface:
 
-      typedef std::function<Command* ()> cmd_maker_fn;
-
-      static std::map<std::string, cmd_maker_fn>& global_registry()
-         {
-         static std::map<std::string, cmd_maker_fn> g_cmds;
-         return g_cmds;
-         }
-
-      static std::unique_ptr<Command> get_cmd(const std::string& name)
-         {
-         auto& reg = Command::global_registry();
-
-         std::unique_ptr<Command> r;
-         auto i = reg.find(name);
-         if(i != reg.end())
-            {
-            r.reset(i->second());
-            }
-
-         return r;
-         }
-
       class Registration final
          {
          public:
-            Registration(const std::string& name, cmd_maker_fn maker_fn)
-               {
-               auto& reg = Command::global_registry();
-
-               if(reg.count(name) > 0)
-                  {
-                  throw CLI_Error("Duplicated registration of command " + name);
-                  }
-
-               Command::global_registry().insert(std::make_pair(name, maker_fn));
-               }
+            Registration(const std::string& name, cmd_maker_fn maker_fn);
          };
    };
 
diff --git a/src/cli/cli_exceptions.h b/src/cli/cli_exceptions.h
new file mode 100644
index 000000000..ed7be3137
--- /dev/null
+++ b/src/cli/cli_exceptions.h
@@ -0,0 +1,44 @@
+/*
+* (C) 2015 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_CLI_EXCEPTIONS_H_
+#define BOTAN_CLI_EXCEPTIONS_H_
+
+namespace Botan_CLI {
+
+class CLI_Error : public std::runtime_error
+   {
+   public:
+      explicit CLI_Error(const std::string& s) : std::runtime_error(s) {}
+   };
+
+class CLI_IO_Error : public CLI_Error
+   {
+   public:
+      CLI_IO_Error(const std::string& op, const std::string& who) :
+         CLI_Error("Error " + op + " " + who) {}
+   };
+
+class CLI_Usage_Error : public CLI_Error
+   {
+   public:
+      explicit CLI_Usage_Error(const std::string& what) : CLI_Error(what) {}
+   };
+
+/* Thrown eg when a requested feature was compiled out of the library
+   or is not available, eg hashing with
+*/
+class CLI_Error_Unsupported : public CLI_Error
+   {
+   public:
+      CLI_Error_Unsupported(const std::string& what,
+                            const std::string& who)
+         : CLI_Error(what + " with '" + who + "' unsupported or not available") {}
+   };
+
+}
+
+#endif
diff --git a/src/cli/cli_rng.cpp b/src/cli/cli_rng.cpp
new file mode 100644
index 000000000..1e3ecb141
--- /dev/null
+++ b/src/cli/cli_rng.cpp
@@ -0,0 +1,89 @@
+/*
+* (C) 2015,2017 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include "cli.h"
+#include <botan/rng.h>
+#include <botan/entropy_src.h>
+#include <botan/cpuid.h>
+#include <botan/hex.h>
+
+#if defined(BOTAN_HAS_AUTO_SEEDING_RNG)
+   #include <botan/auto_rng.h>
+#endif
+
+#if defined(BOTAN_HAS_SYSTEM_RNG)
+   #include <botan/system_rng.h>
+#endif
+
+#if defined(BOTAN_HAS_RDRAND_RNG)
+   #include <botan/rdrand_rng.h>
+#endif
+
+#if defined(BOTAN_HAS_HMAC_DRBG)
+   #include <botan/hmac_drbg.h>
+#endif
+
+namespace Botan_CLI {
+
+std::unique_ptr<Botan::RandomNumberGenerator>
+cli_make_rng(const std::string& rng_type, const std::string& hex_drbg_seed)
+   {
+#if defined(BOTAN_HAS_SYSTEM_RNG)
+   if(rng_type == "system" || rng_type.empty())
+      {
+      return std::unique_ptr<Botan::RandomNumberGenerator>(new Botan::System_RNG);
+      }
+#endif
+
+#if defined(BOTAN_HAS_RDRAND_RNG)
+   if(rng_type == "rdrand")
+      {
+      if(Botan::CPUID::has_rdrand())
+         return std::unique_ptr<Botan::RandomNumberGenerator>(new Botan::RDRAND_RNG);
+      else
+         throw CLI_Error("RDRAND instruction not supported on this processor");
+      }
+#endif
+
+   const std::vector<uint8_t> drbg_seed = Botan::hex_decode(hex_drbg_seed);
+
+#if defined(BOTAN_HAS_AUTO_SEEDING_RNG)
+   if(rng_type == "auto" || rng_type == "entropy" || rng_type.empty())
+      {
+      std::unique_ptr<Botan::RandomNumberGenerator> rng;
+
+      if(rng_type == "entropy")
+         rng.reset(new Botan::AutoSeeded_RNG(Botan::Entropy_Sources::global_sources()));
+      else
+         rng.reset(new Botan::AutoSeeded_RNG);
+
+      if(drbg_seed.size() > 0)
+         rng->add_entropy(drbg_seed.data(), drbg_seed.size());
+      return rng;
+      }
+#endif
+
+#if defined(BOTAN_HAS_HMAC_DRBG) && defined(BOTAN_AUTO_RNG_HMAC)
+   if(rng_type == "drbg")
+      {
+      std::unique_ptr<Botan::MessageAuthenticationCode> mac =
+         Botan::MessageAuthenticationCode::create(BOTAN_AUTO_RNG_HMAC);
+      std::unique_ptr<Botan::Stateful_RNG> rng(new Botan::HMAC_DRBG(std::move(mac)));
+      rng->add_entropy(drbg_seed.data(), drbg_seed.size());
+
+      if(rng->is_seeded() == false)
+         throw CLI_Error("For " + rng->name() + " a seed of at least " +
+                         std::to_string(rng->security_level()/8) +
+                         " bytes must be provided");
+
+      return std::unique_ptr<Botan::RandomNumberGenerator>(rng.release());
+      }
+#endif
+
+   throw CLI_Error_Unsupported("RNG", rng_type);
+   }
+
+}
diff --git a/src/cli/compress.cpp b/src/cli/compress.cpp
index c60a33921..e9cd02290 100644
--- a/src/cli/compress.cpp
+++ b/src/cli/compress.cpp
@@ -8,6 +8,7 @@
 
 #if defined(BOTAN_HAS_COMPRESSION)
    #include <botan/compression.h>
+   #include <fstream>
 #endif
 
 namespace Botan_CLI {
@@ -75,17 +76,17 @@ class Compress final : public Command
          while(in.good())
             {
             buf.resize(buf_size);
-            in.read(reinterpret_cast<char*>(&buf[0]), buf.size());
+            in.read(reinterpret_cast<char*>(buf.data()), buf.size());
             buf.resize(in.gcount());
 
             compress->update(buf);
 
-            out.write(reinterpret_cast<const char*>(&buf[0]), buf.size());
+            out.write(reinterpret_cast<const char*>(buf.data()), buf.size());
             }
 
          buf.clear();
          compress->finish(buf);
-         out.write(reinterpret_cast<const char*>(&buf[0]), buf.size());
+         out.write(reinterpret_cast<const char*>(buf.data()), buf.size());
          out.close();
          }
    };
@@ -147,17 +148,17 @@ class Decompress final : public Command
          while(in.good())
             {
             buf.resize(buf_size);
-            in.read(reinterpret_cast<char*>(&buf[0]), buf.size());
+            in.read(reinterpret_cast<char*>(buf.data()), buf.size());
             buf.resize(in.gcount());
 
             decompress->update(buf);
 
-            out.write(reinterpret_cast<const char*>(&buf[0]), buf.size());
+            out.write(reinterpret_cast<const char*>(buf.data()), buf.size());
             }
 
          buf.clear();
          decompress->finish(buf);
-         out.write(reinterpret_cast<const char*>(&buf[0]), buf.size());
+         out.write(reinterpret_cast<const char*>(buf.data()), buf.size());
          out.close();
          }
    };
diff --git a/src/cli/credentials.h b/src/cli/credentials.h
index 38f91141e..3b46c239c 100644
--- a/src/cli/credentials.h
+++ b/src/cli/credentials.h
@@ -11,7 +11,6 @@
 #include <botan/credentials_manager.h>
 #include <botan/x509self.h>
 #include <botan/data_src.h>
-#include <fstream>
 #include <memory>
 
 inline bool value_exists(const std::vector<std::string>& vec,
diff --git a/src/cli/encryption.cpp b/src/cli/encryption.cpp
index e8d5622c2..ffc1d0b48 100644
--- a/src/cli/encryption.cpp
+++ b/src/cli/encryption.cpp
@@ -9,7 +9,6 @@
 #if defined(BOTAN_HAS_AES) && defined(BOTAN_HAS_AEAD_MODES)
 
 #include <botan/aead.h>
-#include <iterator>
 #include <sstream>
 
 using namespace Botan;
@@ -39,7 +38,7 @@ bool is_aead(const std::string &cipher)
    }
 
 secure_vector<byte> do_crypt(const std::string &cipher,
-                             const secure_vector<byte> &input,
+                             const std::vector<byte> &input,
                              const SymmetricKey &key,
                              const InitializationVector &iv,
                              const OctetString &ad,
@@ -97,7 +96,7 @@ class Encryption final : public Command
          const std::string ad_hex  = get_arg_or("ad", "");
          const size_t buf_size = get_arg_sz("buf-size");
 
-         Botan::secure_vector<uint8_t> input = this->slurp_file_locked("-", buf_size);
+         const std::vector<uint8_t> input = this->slurp_file("-", buf_size);
 
          if (verbose())
             {
@@ -109,8 +108,7 @@ class Encryption final : public Command
          auto ad = OctetString(ad_hex);
 
          auto direction = flag_set("decrypt") ? Cipher_Dir::DECRYPTION : Cipher_Dir::ENCRYPTION;
-         auto data = do_crypt(VALID_MODES[mode], input, key, iv, ad, direction);
-         std::copy(data.begin(), data.end(), std::ostreambuf_iterator<char>(output()));
+         write_output(do_crypt(VALID_MODES[mode], input, key, iv, ad, direction));
          }
    };
 
diff --git a/src/cli/main.cpp b/src/cli/main.cpp
index 5fbaf085e..6b0024465 100644
--- a/src/cli/main.cpp
+++ b/src/cli/main.cpp
@@ -7,25 +7,21 @@
 #include "cli.h"
 
 #include <botan/version.h>
-#include <botan/internal/stl_util.h>
-#include <iterator>
 #include <sstream>
+#include <iostream>
 
 namespace {
 
 std::string main_help()
    {
-   const std::set<std::string> avail_commands =
-      Botan::map_keys_as_set(Botan_CLI::Command::global_registry());
-
    std::ostringstream oss;
 
    oss << "Usage: botan <cmd> <cmd-options>\n";
    oss << "Available commands:\n";
 
-   for(auto& cmd_name : avail_commands)
+   for(const auto& cmd_name : Botan_CLI::Command::registered_cmds())
       {
-      auto cmd = Botan_CLI::Command::get_cmd(cmd_name);
+      std::unique_ptr<Botan_CLI::Command> cmd = Botan_CLI::Command::get_cmd(cmd_name);
       oss << cmd->cmd_spec() << "\n";
       }
 
diff --git a/src/cli/speed.cpp b/src/cli/speed.cpp
index 518adb857..e075e752d 100644
--- a/src/cli/speed.cpp
+++ b/src/cli/speed.cpp
@@ -23,6 +23,7 @@
 #include <botan/mac.h>
 #include <botan/cipher_mode.h>
 #include <botan/entropy_src.h>
+#include <botan/parsing.h>
 #include <botan/cpuid.h>
 #include <botan/internal/os_utils.h>
 #include <botan/version.h>
diff --git a/src/cli/timing_tests.cpp b/src/cli/timing_tests.cpp
index 258780845..d95df026c 100644
--- a/src/cli/timing_tests.cpp
+++ b/src/cli/timing_tests.cpp
@@ -20,6 +20,8 @@
 #include "cli.h"
 #include <botan/hex.h>
 #include <sstream>
+#include <fstream>
+
 #include <botan/internal/os_utils.h>
 
 #if defined(BOTAN_HAS_SYSTEM_RNG)
diff --git a/src/cli/tls_client.cpp b/src/cli/tls_client.cpp
index d773cd81f..db0453a72 100644
--- a/src/cli/tls_client.cpp
+++ b/src/cli/tls_client.cpp
@@ -15,6 +15,8 @@
 #include <botan/x509path.h>
 #include <botan/ocsp.h>
 #include <botan/hex.h>
+#include <botan/parsing.h>
+#include <fstream>
 
 #if defined(BOTAN_HAS_TLS_SQLITE3_SESSION_MANAGER)
    #include <botan/tls_session_manager_sqlite.h>
diff --git a/src/cli/tls_server.cpp b/src/cli/tls_server.cpp
index 703967641..02c15bf7b 100644
--- a/src/cli/tls_server.cpp
+++ b/src/cli/tls_server.cpp
@@ -17,6 +17,7 @@
 #include "credentials.h"
 
 #include <list>
+#include <fstream>
 
 #if defined(BOTAN_TARGET_OS_IS_WINDOWS)
 #include <winsock2.h>
diff --git a/src/cli/tls_utils.cpp b/src/cli/tls_utils.cpp
index f62781af9..7b2474a69 100644
--- a/src/cli/tls_utils.cpp
+++ b/src/cli/tls_utils.cpp
@@ -114,13 +114,8 @@ class TLS_Ciphersuites final : public Command
             }
          else
             {
-            std::ifstream policy_file(policy_type);
-            if(!policy_file.good())
-               {
-               throw CLI_Error("Error TLS policy '" + policy_type + "' is neither a file nor a known policy type");
-               }
-
-            policy.reset(new Botan::TLS::Text_Policy(policy_file));
+            const std::string policy_txt = slurp_file_as_str(policy_type);
+            policy.reset(new Botan::TLS::Text_Policy(policy_txt));
             }
 
          for(uint16_t suite_id : policy->ciphersuite_list(version, with_srp))
diff --git a/src/cli/utils.cpp b/src/cli/utils.cpp
index 6b108eb43..1e2ddf42e 100644
--- a/src/cli/utils.cpp
+++ b/src/cli/utils.cpp
@@ -10,30 +10,15 @@
 #include <botan/version.h>
 #include <botan/hash.h>
 #include <botan/mac.h>
+#include <botan/rng.h>
 #include <botan/cpuid.h>
 #include <botan/hex.h>
-#include <botan/entropy_src.h>
+#include <botan/parsing.h>
 
 #if defined(BOTAN_HAS_BASE64_CODEC)
    #include <botan/base64.h>
 #endif
 
-#if defined(BOTAN_HAS_AUTO_SEEDING_RNG)
-   #include <botan/auto_rng.h>
-#endif
-
-#if defined(BOTAN_HAS_SYSTEM_RNG)
-   #include <botan/system_rng.h>
-#endif
-
-#if defined(BOTAN_HAS_RDRAND_RNG)
-   #include <botan/rdrand_rng.h>
-#endif
-
-#if defined(BOTAN_HAS_HMAC_DRBG)
-   #include <botan/hmac_drbg.h>
-#endif
-
 #if defined(BOTAN_HAS_HTTP_UTIL)
    #include <botan/http_util.h>
 #endif
@@ -44,64 +29,6 @@
 
 namespace Botan_CLI {
 
-std::unique_ptr<Botan::RandomNumberGenerator>
-cli_make_rng(const std::string& rng_type, const std::string& hex_drbg_seed)
-   {
-#if defined(BOTAN_HAS_SYSTEM_RNG)
-   if(rng_type == "system" || rng_type.empty())
-      {
-      return std::unique_ptr<Botan::RandomNumberGenerator>(new Botan::System_RNG);
-      }
-#endif
-
-#if defined(BOTAN_HAS_RDRAND_RNG)
-   if(rng_type == "rdrand")
-      {
-      if(Botan::CPUID::has_rdrand())
-         return std::unique_ptr<Botan::RandomNumberGenerator>(new Botan::RDRAND_RNG);
-      else
-         throw CLI_Error("RDRAND instruction not supported on this processor");
-      }
-#endif
-
-   const std::vector<uint8_t> drbg_seed = Botan::hex_decode(hex_drbg_seed);
-
-#if defined(BOTAN_HAS_AUTO_SEEDING_RNG)
-   if(rng_type == "auto" || rng_type == "entropy" || rng_type.empty())
-      {
-      std::unique_ptr<Botan::RandomNumberGenerator> rng;
-
-      if(rng_type == "entropy")
-         rng.reset(new Botan::AutoSeeded_RNG(Botan::Entropy_Sources::global_sources()));
-      else
-         rng.reset(new Botan::AutoSeeded_RNG);
-
-      if(drbg_seed.size() > 0)
-         rng->add_entropy(drbg_seed.data(), drbg_seed.size());
-      return rng;
-      }
-#endif
-
-#if defined(BOTAN_HAS_HMAC_DRBG) && defined(BOTAN_AUTO_RNG_HMAC)
-   if(rng_type == "drbg")
-      {
-      std::unique_ptr<Botan::MessageAuthenticationCode> mac =
-         Botan::MessageAuthenticationCode::create(BOTAN_AUTO_RNG_HMAC);
-      std::unique_ptr<Botan::Stateful_RNG> rng(new Botan::HMAC_DRBG(std::move(mac)));
-      rng->add_entropy(drbg_seed.data(), drbg_seed.size());
-
-      if(rng->is_seeded() == false)
-         throw CLI_Error("For " + rng->name() + " a seed of at least " +
-                         std::to_string(rng->security_level()/8) +
-                         " bytes must be provided");
-
-      return std::unique_ptr<Botan::RandomNumberGenerator>(rng.release());
-      }
-#endif
-
-   throw CLI_Error_Unsupported("RNG", rng_type);
-   }
-
 class Config_Info final : public Command
    {
    public:
@@ -245,7 +172,7 @@ class RNG final : public Command
             }
 
          const std::string drbg_seed = get_arg("drbg-seed");
-         std::unique_ptr<Botan::RNG> rng = cli_make_rng(type, drbg_seed);
+         std::unique_ptr<Botan::RandomNumberGenerator> rng = cli_make_rng(type, drbg_seed);
 
          for(const std::string& req : get_arg_list("bytes"))
             {