diff options
| -rw-r--r-- | include/elevator/IOUtil.hpp | 62 | ||||
| -rwxr-xr-x | scripts/test_exe_template.sh | 4 | ||||
| -rw-r--r-- | src/elevator/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | src/elevator/IOUtil.cpp | 289 | ||||
| -rw-r--r-- | test/elevator/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | test/elevator/test_02_iostream.cpp | 119 | ||||
| -rw-r--r-- | test/elevator/test_data.cpp | 13 | ||||
| -rw-r--r-- | test/elevator/test_data.hpp | 30 |
8 files changed, 476 insertions, 47 deletions
diff --git a/include/elevator/IOUtil.hpp b/include/elevator/IOUtil.hpp index d4d8293..92b47e6 100644 --- a/include/elevator/IOUtil.hpp +++ b/include/elevator/IOUtil.hpp @@ -4,12 +4,14 @@ */ #ifndef IOUTIL_HPP_ -#define IOUtil_HPP_ +#define IOUTIL_HPP_ #include <string> #include <cstdint> +#include <functional> #include <jau/basic_types.hpp> +#include <jau/ringbuffer.hpp> #include <botan_all.h> @@ -17,11 +19,28 @@ namespace elevator { class IOUtil { public: - static bool file_exists(const std::string& name); - static bool remove(const std::string& fname); + static bool file_exists(const std::string& name) noexcept; + static bool remove(const std::string& fname) noexcept; typedef std::function<void (Botan::secure_vector<uint8_t>& /* data */, bool /* is_final */)> StreamConsumerFunc; + typedef jau::ringbuffer<uint8_t, uint8_t, size_t> ByteRingbuffer; + + /** + * Operation result value + */ + enum class result_t : int8_t { + /** Operation failed. */ + FAILED = -1, + + /** Operation still in progress. */ + NONE = 0, + + /** Operation succeeded. */ + SUCCESS = 1 + }; + typedef jau::ordered_atomic<result_t, std::memory_order::memory_order_relaxed> relaxed_atomic_result_t; + /** * * @param input_file @@ -51,10 +70,45 @@ class IOUtil { static ssize_t read_stream(Botan::DataSource& in, Botan::secure_vector<uint8_t>& buffer, StreamConsumerFunc consumer_fn); + /** + * + * @param url + * @param buffer + * @param consumer_fn + * @return total bytes read or -1 if error + */ static ssize_t read_http_get(const std::string& url, Botan::secure_vector<uint8_t>& buffer, StreamConsumerFunc consumer_fn); - static void print_stats(const std::string &prefix, const uint64_t out_bytes_total, uint64_t td_ms); + static const size_t BEST_HTTP_RINGBUFFER_SIZE; + + /** + * Asynchronous http read content using a byte jau::ringbuffer, + * allowing parallel reading. + * + * @param url the URL of the content to read + * @param buffer the ringbuffer destination to write into + * @param content_length tracking the content_length + * @param total_read tracking the total_read + * @param result tracking the result_t + */ + static void read_http_get(const std::string& url, ByteRingbuffer& buffer, + jau::relaxed_atomic_ssize_t& content_length, + jau::relaxed_atomic_ssize_t& total_read, + relaxed_atomic_result_t& result) noexcept; + + /** + * Asynchronous http read content using a byte jau::ringbuffer, + * allowing parallel reading. + * + * @param url the URL of the content to read + * @param buffer the ringbuffer destination to write into + * @param result tracking the result_t + */ + static void read_http_get(const std::string& url, ByteRingbuffer& buffer, + relaxed_atomic_result_t& result) noexcept; + + static void print_stats(const std::string &prefix, const uint64_t out_bytes_total, uint64_t td_ms) noexcept; }; } // namespace elevator diff --git a/scripts/test_exe_template.sh b/scripts/test_exe_template.sh index 3f63af9..c7a0214 100755 --- a/scripts/test_exe_template.sh +++ b/scripts/test_exe_template.sh @@ -1,5 +1,8 @@ #!/bin/sh +export elevator_debug=true +export elevator_verbose=true + # Arguments: # --perf_analysis special performance analysis using 3rd party tools # -v normal dummy for full benchmarking @@ -67,6 +70,7 @@ runit() { #export ASAN_OPTIONS=verbosity=1:malloc_context_size=20 #export ASAN_OPTIONS=print_stats:halt_on_error:replace_intrin + cd $build_dir/test/elevator/ $EXE_WRAPPER $build_dir/test/elevator/$bname $* } diff --git a/src/elevator/CMakeLists.txt b/src/elevator/CMakeLists.txt index bd0037b..287bee2 100644 --- a/src/elevator/CMakeLists.txt +++ b/src/elevator/CMakeLists.txt @@ -32,10 +32,10 @@ target_link_libraries ( ${CMAKE_THREAD_LIBS_INIT} ) -if(CMAKE_COMPILER_IS_GNUCC) +# if(CMAKE_COMPILER_IS_GNUCC) # botan fails on cast align etc target_compile_options(elevator PUBLIC "-Wno-cast-align" "-Wno-unused-parameter") -endif(CMAKE_COMPILER_IS_GNUCC) +# endif(CMAKE_COMPILER_IS_GNUCC) if(USE_STRIP) add_custom_command(TARGET elevator POST_BUILD diff --git a/src/elevator/IOUtil.cpp b/src/elevator/IOUtil.cpp index 205cbad..6d47fd6 100644 --- a/src/elevator/IOUtil.cpp +++ b/src/elevator/IOUtil.cpp @@ -5,6 +5,7 @@ #include <fstream> #include <iostream> +#include <chrono> #include <elevator/elevator.hpp> @@ -14,6 +15,9 @@ #include <curl/curl.h> +#include <thread> +#include <pthread.h> + // #define USE_CXX17lib_FS 1 #if USE_CXX17lib_FS #include <filesystem> @@ -22,12 +26,12 @@ using namespace elevator; -bool IOUtil::file_exists(const std::string& name) { +bool IOUtil::file_exists(const std::string& name) noexcept { std::ifstream f(name); return f.good() && f.is_open(); } -bool IOUtil::remove(const std::string& fname) { +bool IOUtil::remove(const std::string& fname) noexcept { #if USE_CXX17lib_FS const fs::path fname2 = fname; return fs::remove(fname); @@ -87,7 +91,7 @@ ssize_t IOUtil::read_stream(Botan::DataSource& in, Botan::secure_vector<uint8_t> return total; } -struct curl_glue_t { +struct curl_glue1_t { CURL *curl_handle; ssize_t content_length; ssize_t total_read; @@ -95,8 +99,8 @@ struct curl_glue_t { IOUtil::StreamConsumerFunc consumer_fn; }; -static size_t consume_curl0(void *ptr, size_t size, size_t nmemb, void *stream) { - curl_glue_t * cg = (curl_glue_t*)stream; +static size_t consume_curl1(void *ptr, size_t size, size_t nmemb, void *stream) noexcept { + curl_glue1_t * cg = (curl_glue1_t*)stream; if( 0 > cg->content_length ) { curl_off_t v = 0; @@ -110,7 +114,8 @@ static size_t consume_curl0(void *ptr, size_t size, size_t nmemb, void *stream) memcpy(cg->buffer.data(), ptr, realsize); cg->total_read += realsize; - bool is_final = 0 < cg->content_length ? cg->total_read >= cg->content_length : false; + const bool is_final = 0 == realsize || + ( 0 < cg->content_length ) ? cg->total_read >= cg->content_length : false; cg->consumer_fn(cg->buffer, is_final); return realsize; @@ -118,36 +123,270 @@ static size_t consume_curl0(void *ptr, size_t size, size_t nmemb, void *stream) ssize_t IOUtil::read_http_get(const std::string& url, Botan::secure_vector<uint8_t>& buffer, StreamConsumerFunc consumer_fn) { - /* init the curl session */ - CURL *curl_handle = curl_easy_init(); + std::vector<char> errorbuffer; + errorbuffer.reserve(CURL_ERROR_SIZE); + CURLcode res; + + /* init the curl session */ + CURL *curl_handle = curl_easy_init(); + if( nullptr == curl_handle ) { + ERR_PRINT("Error setting up http url %s, null curl handle", url.c_str()); + return -1; + } + + curl_glue1_t cg = { curl_handle, -1, 0, buffer, consumer_fn }; + + res = curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, errorbuffer.data()); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url.c_str(), (int)res, curl_easy_strerror(res)); + goto errout; + } + + /* set URL to get here */ + res = curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str()); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url.c_str(), (int)res, errorbuffer.data()); + goto errout; + } + + /* Switch on full protocol/debug output while testing */ + res = curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 0L); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url.c_str(), (int)res, errorbuffer.data()); + goto errout; + } + + /* disable progress meter, set to 0L to enable it */ + res = curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url.c_str(), (int)res, errorbuffer.data()); + goto errout; + } + + /* send all data to this function */ + res = curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, consume_curl1); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url.c_str(), (int)res, errorbuffer.data()); + goto errout; + } + + /* write the page body to this file handle */ + res = curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)&cg); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url.c_str(), (int)res, errorbuffer.data()); + goto errout; + } + + /* performs the tast, blocking! */ + res = curl_easy_perform(curl_handle); + if( CURLE_OK != res ) { + ERR_PRINT("Error processing http url %s, error %d %d", + url.c_str(), (int)res, errorbuffer.data()); + goto errout; + } + + /* cleanup curl stuff */ + curl_easy_cleanup(curl_handle); + return cg.total_read; + +errout: + curl_easy_cleanup(curl_handle); + return -1; +} + +struct curl_glue2_t { + curl_glue2_t(CURL *_curl_handle, + jau::relaxed_atomic_ssize_t* _content_length, + bool _content_length_mine, + jau::relaxed_atomic_ssize_t* _total_read, + bool _total_read_mine, + IOUtil::ByteRingbuffer& _buffer, + IOUtil::relaxed_atomic_result_t& _result) + : curl_handle(_curl_handle), + content_length(_content_length), + content_length_mine(_content_length_mine), + total_read(_total_read), + total_read_mine(_total_read_mine), + buffer(_buffer), + result(_result) + {} + + CURL *curl_handle; + jau::relaxed_atomic_ssize_t* content_length; + bool content_length_mine; + jau::relaxed_atomic_ssize_t* total_read; + bool total_read_mine; + IOUtil::ByteRingbuffer& buffer; + IOUtil::relaxed_atomic_result_t& result; +}; + +static size_t consume_curl2(void *ptr, size_t size, size_t nmemb, void *stream) noexcept { + curl_glue2_t * cg = (curl_glue2_t*)stream; + + if( 0 > *cg->content_length ) { + curl_off_t v = 0; + const CURLcode r = curl_easy_getinfo(cg->curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &v); + if( CURLE_OK == r ) { + *cg->content_length = v; + } + } + ssize_t total_read = *cg->total_read; + const size_t realsize = size * nmemb; + DBG_PRINT("consume_curl2.0 realsize % " PRIu64 ", rb %s", realsize, cg->buffer.toString().c_str() ); + cg->buffer.putBlocking(reinterpret_cast<uint8_t*>(ptr), + reinterpret_cast<uint8_t*>(ptr)+realsize, 0 /* timeoutMS */); + + total_read += realsize; + *cg->total_read = total_read; + const bool is_final = 0 == realsize || + ( 0 < *cg->content_length ) ? total_read >= *cg->content_length : false; + if( is_final ) { + cg->result = IOUtil::result_t::SUCCESS; + } + + DBG_PRINT("consume_curl2.X realsize % " PRIu64 ", total %" PRIi64 ", result %d, rb %s", + realsize, total_read, cg->result.load(), cg->buffer.toString().c_str() ); - curl_glue_t cg = { curl_handle, -1, 0, buffer, consumer_fn }; + return realsize; +} + +static void read_http_get_thread(const char *url, std::unique_ptr<curl_glue2_t> && cg) noexcept { + std::vector<char> errorbuffer; + errorbuffer.reserve(CURL_ERROR_SIZE); + CURLcode res; + + /* init the curl session */ + CURL *curl_handle = curl_easy_init(); + if( nullptr == curl_handle ) { + ERR_PRINT("Error setting up http url %s, null curl handle", url); + goto errout; + } + cg->curl_handle = curl_handle; + + res = curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, errorbuffer.data()); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url, (int)res, curl_easy_strerror(res)); + goto errout; + } + + /* set URL to get here */ + res = curl_easy_setopt(curl_handle, CURLOPT_URL, url); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url, (int)res, errorbuffer.data()); + goto errout; + } + + /* Switch on full protocol/debug output while testing */ + res = curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 0L); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url, (int)res, errorbuffer.data()); + goto errout; + } + + /* disable progress meter, set to 0L to enable it */ + res = curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url, (int)res, errorbuffer.data()); + goto errout; + } + + /* send all data to this function */ + res = curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, consume_curl2); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url, (int)res, errorbuffer.data()); + goto errout; + } + + /* write the page body to this file handle */ + res = curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)cg.get()); + if( CURLE_OK != res ) { + ERR_PRINT("Error setting up http url %s, error %d %d", + url, (int)res, errorbuffer.data()); + goto errout; + } + + /* performs the tast, blocking! */ + res = curl_easy_perform(curl_handle); + if( CURLE_OK != res ) { + ERR_PRINT("Error processing http url %s, error %d %d", + url, (int)res, errorbuffer.data()); + goto errout; + } + + /* cleanup curl stuff */ + cg->result = IOUtil::result_t::SUCCESS; + goto cleanup; + +errout: + cg->result = IOUtil::result_t::FAILED; + +cleanup: + if( nullptr != curl_handle ) { + curl_easy_cleanup(curl_handle); + } + + if( cg->content_length_mine ) { + delete cg->content_length; + cg->content_length = nullptr; + } + if( cg->total_read_mine ) { + delete cg->total_read; + cg->total_read = nullptr; + } + return; +} + +const size_t IOUtil::BEST_HTTP_RINGBUFFER_SIZE = 2*CURL_MAX_WRITE_SIZE; - /* set URL to get here */ - curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str()); +void IOUtil::read_http_get(const std::string& url, ByteRingbuffer& buffer, + jau::relaxed_atomic_ssize_t& content_length, + jau::relaxed_atomic_ssize_t& total_read, + relaxed_atomic_result_t& result) noexcept { + /* init user referenced values */ + content_length = -1; + total_read = 0; + result = IOUtil::result_t::NONE; - /* Switch on full protocol/debug output while testing */ - curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 0L); + if( buffer.capacity() < BEST_HTTP_RINGBUFFER_SIZE ) { + buffer.recapacity( BEST_HTTP_RINGBUFFER_SIZE ); + } + + std::unique_ptr<curl_glue2_t> cg ( std::make_unique<curl_glue2_t>(nullptr, &content_length, false, &total_read, false, buffer, result ) ); - /* disable progress meter, set to 0L to enable it */ - curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L); + std::thread http_thread00(&::read_http_get_thread, url.c_str(), std::move(cg)); // @suppress("Invalid arguments") + http_thread00.detach(); +} - /* send all data to this function */ - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, consume_curl0); +void IOUtil::read_http_get(const std::string& url, ByteRingbuffer& buffer, + relaxed_atomic_result_t& result) noexcept { - /* write the page body to this file handle */ - curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)&cg); + /* init user referenced values */ + result = IOUtil::result_t::NONE; - /* performs the tast, blocking! */ - curl_easy_perform(curl_handle); + if( buffer.capacity() < BEST_HTTP_RINGBUFFER_SIZE ) { + buffer.recapacity( BEST_HTTP_RINGBUFFER_SIZE ); + } - /* cleanup curl stuff */ - curl_easy_cleanup(curl_handle); + jau::relaxed_atomic_ssize_t* content_length = new jau::relaxed_atomic_ssize_t(-1); + jau::relaxed_atomic_ssize_t* total_read = new jau::relaxed_atomic_ssize_t(0); + std::unique_ptr<curl_glue2_t> cg ( std::make_unique<curl_glue2_t>(nullptr, content_length, true, total_read, true, buffer, result ) ); - return cg.total_read; + std::thread http_thread00(&::read_http_get_thread, url.c_str(), std::move(cg)); // @suppress("Invalid arguments") + http_thread00.detach(); } -void IOUtil::print_stats(const std::string &prefix, const uint64_t out_bytes_total, uint64_t td_ms) { +void IOUtil::print_stats(const std::string &prefix, const uint64_t out_bytes_total, uint64_t td_ms) noexcept { if( jau::environment::get().verbose ) { jau::PLAIN_PRINT(true, "%s: Duration %s s, %s ms", prefix.c_str(), diff --git a/test/elevator/CMakeLists.txt b/test/elevator/CMakeLists.txt index 28c07ad..751747a 100644 --- a/test/elevator/CMakeLists.txt +++ b/test/elevator/CMakeLists.txt @@ -25,7 +25,7 @@ string( REPLACE ".cpp" "" BASENAMES_IDIOMATIC_EXAMPLES "${SOURCES_IDIOMATIC_EXAM set( TARGETS_IDIOMATIC_EXAMPLES ${BASENAMES_IDIOMATIC_EXAMPLES} ) foreach( name ${TARGETS_IDIOMATIC_EXAMPLES} ) - add_executable(${name} ${name}.cpp) + add_executable(${name} ${name}.cpp test_data.cpp) target_link_libraries(${name} elevator catch2l) add_dependencies(${name} elevator catch2l) add_test (NAME ${name} COMMAND ${name}) diff --git a/test/elevator/test_02_iostream.cpp b/test/elevator/test_02_iostream.cpp index 7a7d112..e2b66a5 100644 --- a/test/elevator/test_02_iostream.cpp +++ b/test/elevator/test_02_iostream.cpp @@ -11,6 +11,9 @@ #include <fstream> #include <iostream> +#include <thread> +#include <pthread.h> + // #define CATCH_CONFIG_RUNNER #define CATCH_CONFIG_MAIN #include <catch2/catch_amalgamated.hpp> @@ -18,6 +21,8 @@ #include <elevator/elevator.hpp> +#include "test_data.hpp" + #include <jau/debug.hpp> extern "C" { @@ -26,21 +31,105 @@ extern "C" { using namespace elevator; -TEST_CASE( "Elevator Test 01 IOStream", "[io][stream]" ) { +class Test02IOStream : public TestData { + public: + static void test01() { + const std::string url_input = url_input_root + basename_64kB + ".enc"; - // const std::string url_input("http://jordan/deployment/data-382MB.mkv.enc"); - const std::string url_input("http://jordan/deployment/data-64kB.bin.enc"); + std::ofstream outfile("test02_01_out.bin", std::ios::out | std::ios::binary); + REQUIRE( outfile.good() ); + REQUIRE( outfile.is_open() ); - Botan::secure_vector<uint8_t> buffer(4096); - ssize_t calls0 = 0; - ssize_t total0 = 0; - auto consume = [&](Botan::secure_vector<uint8_t>& data, bool is_final) noexcept { - calls0++; - total0 += data.size(); - jau::PLAIN_PRINT("test", "#% " PRIi64 ": consumed size % " PRIu64 ", total %" PRIi64 ", capacity %" PRIu64 ", final %d", - calls0, data.size(), total0, data.capacity(), is_final ); - }; - ssize_t total1 = IOUtil::read_http_get(url_input, buffer, consume); + Botan::secure_vector<uint8_t> buffer(4096); + ssize_t consumed_calls = 0; + ssize_t consumed_total_bytes = 0; + auto consume = [&](Botan::secure_vector<uint8_t>& data, bool is_final) noexcept { + consumed_calls++; + consumed_total_bytes += data.size(); + outfile.write(reinterpret_cast<char*>(data.data()), data.size()); + jau::PLAIN_PRINT(true, "test02io01 #% " PRIi64 ": consumed size % " PRIu64 ", total %" PRIi64 ", capacity %" PRIu64 ", final %d", + consumed_calls, data.size(), consumed_total_bytes, data.capacity(), is_final ); + }; + ssize_t http_total_bytes = IOUtil::read_http_get(url_input, buffer, consume); + const ssize_t out_bytes_total = outfile.tellp(); + jau::PLAIN_PRINT(true, "test02io01 Done: total %" PRIi64 ", capacity %" PRIu64, + consumed_total_bytes, buffer.capacity()); - REQUIRE( total0 == total1 ); -} + REQUIRE( consumed_total_bytes == http_total_bytes ); + REQUIRE( consumed_total_bytes == out_bytes_total ); + } + + static void test02() { + const std::string url_input = url_input_root + basename_64kB + ".enc"; + + std::ofstream outfile("test02_02_out.bin", std::ios::out | std::ios::binary); + REQUIRE( outfile.good() ); + REQUIRE( outfile.is_open() ); + + constexpr const size_t buffer_size = 4096; + IOUtil::ByteRingbuffer rb(0x00, IOUtil::BEST_HTTP_RINGBUFFER_SIZE); + jau::relaxed_atomic_ssize_t content_length; + jau::relaxed_atomic_ssize_t http_total_bytes; + IOUtil::relaxed_atomic_result_t result; + + IOUtil::read_http_get(url_input, rb, content_length, http_total_bytes, result); + + Botan::secure_vector<uint8_t> buffer(buffer_size); + ssize_t consumed_loops = 0; + size_t consumed_total_bytes = 0; + + while( IOUtil::result_t::NONE == result || !rb.isEmpty() ) { + consumed_loops++; + // const size_t consumed_bytes = content_length >= 0 ? std::min(buffer_size, content_length - consumed_total_bytes) : rb.getSize(); + const size_t consumed_bytes = rb.getBlocking(buffer.data(), buffer_size, 1, 0 /* timeoutMS */); + consumed_total_bytes += consumed_bytes; + jau::PLAIN_PRINT(true, "test02io02.0 #% " PRIi64 ": consumed size % " PRIu64 ", total %" PRIi64 ", result %d, rb %s", + consumed_loops, consumed_bytes, consumed_total_bytes, result.load(), rb.toString().c_str() ); + outfile.write(reinterpret_cast<char*>(buffer.data()), consumed_bytes); + } + const ssize_t out_bytes_total = outfile.tellp(); + jau::PLAIN_PRINT(true, "test02io02.X Done: total %" PRIi64 ", result %d, rb %s", + consumed_total_bytes, result.load(), rb.toString().c_str() ); + + REQUIRE( content_length == http_total_bytes ); + REQUIRE( content_length == out_bytes_total ); + REQUIRE( IOUtil::result_t::SUCCESS == result ); + } + + static void test03() { + const std::string url_input = url_input_root + basename_64kB + ".enc"; + + std::ofstream outfile("test02_02_out.bin", std::ios::out | std::ios::binary); + REQUIRE( outfile.good() ); + REQUIRE( outfile.is_open() ); + + constexpr const size_t buffer_size = 4096; + IOUtil::ByteRingbuffer rb(0x00, IOUtil::BEST_HTTP_RINGBUFFER_SIZE); + IOUtil::relaxed_atomic_result_t result; + + IOUtil::read_http_get(url_input, rb, result); + + Botan::secure_vector<uint8_t> buffer(buffer_size); + ssize_t consumed_loops = 0; + size_t consumed_total_bytes = 0; + + while( IOUtil::result_t::NONE == result || !rb.isEmpty() ) { + consumed_loops++; + const size_t consumed_bytes = rb.getBlocking(buffer.data(), buffer_size, 1, 0 /* timeoutMS */); + consumed_total_bytes += consumed_bytes; + jau::PLAIN_PRINT(true, "test02io03.0 #% " PRIi64 ": consumed size % " PRIu64 ", total %" PRIi64 ", result %d, rb %s", + consumed_loops, consumed_bytes, consumed_total_bytes, result.load(), rb.toString().c_str() ); + outfile.write(reinterpret_cast<char*>(buffer.data()), consumed_bytes); + } + const ssize_t out_bytes_total = outfile.tellp(); + jau::PLAIN_PRINT(true, "test02io03.X Done: total %" PRIi64 ", result %d, rb %s", + consumed_total_bytes, result.load(), rb.toString().c_str() ); + + REQUIRE( 0 < out_bytes_total ); + REQUIRE( IOUtil::result_t::SUCCESS == result ); + } +}; + +METHOD_AS_TEST_CASE( Test02IOStream::test01, "Elevator Test 02 IOStream 01"); +METHOD_AS_TEST_CASE( Test02IOStream::test02, "Elevator Test 02 IOStream 02"); +METHOD_AS_TEST_CASE( Test02IOStream::test03, "Elevator Test 02 IOStream 03"); diff --git a/test/elevator/test_data.cpp b/test/elevator/test_data.cpp new file mode 100644 index 0000000..14a3aac --- /dev/null +++ b/test/elevator/test_data.cpp @@ -0,0 +1,13 @@ +#include "test_data.hpp" + +const std::string TestData::enc_pub_key_fname = "../../../keys/terminal_rsa.pub.pem"; +const std::string TestData::dec_sec_key_fname = "../../../keys/terminal_rsa"; +const std::string TestData::dec_sec_key_passphrase = ""; +const std::string TestData::sign_pub_key_fname = "../../../keys/host_rsa.pub.pem"; +const std::string TestData::sign_sec_key_fname = "../../../keys/host_rsa"; +const std::string TestData::sign_sec_key_passphrase = ""; + +const std::string TestData::url_input_root = "http://jordan/deployment/elevator/"; +const std::string TestData::basename_64kB = "data-64kB.bin"; // + '.enc' for encrypted +const std::string TestData::basename_382MB = "data-382MB.mkv"; // + '.enc' for encrypted +const std::string TestData::basename_1GB = "data-1GB.mkv"; // + '.enc' for encrypted diff --git a/test/elevator/test_data.hpp b/test/elevator/test_data.hpp new file mode 100644 index 0000000..542787b --- /dev/null +++ b/test/elevator/test_data.hpp @@ -0,0 +1,30 @@ +/* + * Author: Sven Gothel <[email protected]> + * Copyright (c) 2020 ZAFENA AB + */ + +#ifndef TEST_DATA_HPP_ +#define TEST_DATA_HPP_ + +#include <iostream> +#include <cassert> +#include <cinttypes> +#include <cstring> + +class TestData { + public: + static constexpr const bool overwrite = true; + static const std::string enc_pub_key_fname; + static const std::string dec_sec_key_fname; + static const std::string dec_sec_key_passphrase; + static const std::string sign_pub_key_fname; + static const std::string sign_sec_key_fname; + static const std::string sign_sec_key_passphrase; + + static const std::string url_input_root; + static const std::string basename_64kB; + static const std::string basename_382MB; + static const std::string basename_1GB; +}; + +#endif /* TEST_DATA_HPP_ */ |