diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..6e19512a0e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.dockerignore +Dockerfile diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..4207900484 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +python: + install: + - requirements: doc/manuals/rtd_requirement.txt + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: doc/manuals/conf.py diff --git a/.uncrustify.cfg b/.uncrustify.cfg index ae81276ddc..896aac72ee 100644 --- a/.uncrustify.cfg +++ b/.uncrustify.cfg @@ -3,7 +3,7 @@ using 0.69 # General options newlines = auto -input_tab_size = 8 +input_tab_size = 2 string_escape_char = 92 string_escape_char2 = 0 @@ -70,6 +70,7 @@ sp_permit_cpp11_shift = true sp_before_sparen = remove sp_inside_sparen = force sp_after_sparen = force +sp_inside_for = force sp_sparen_brace = force sp_special_semi = ignore sp_before_semi = remove @@ -77,7 +78,7 @@ sp_before_semi_for = remove sp_before_semi_for_empty = remove sp_after_semi = add sp_after_semi_for = force -sp_after_semi_for_empty = remove +sp_after_semi_for_empty = add sp_before_square = remove sp_before_squares = remove sp_cpp_before_struct_binding = add @@ -100,7 +101,7 @@ sp_after_operator = remove sp_after_operator_sym = remove sp_after_operator_sym_empty = remove sp_after_cast = add -sp_inside_paren_cast = remove +sp_inside_paren_cast = add sp_cpp_cast_paren = remove sp_sizeof_paren = remove sp_sizeof_ellipsis = remove @@ -128,6 +129,7 @@ sp_fparen_brace = add sp_fparen_brace_initializer = remove sp_func_call_paren = remove sp_func_call_paren_empty = remove +sp_func_call_user_paren = remove sp_func_class_paren = remove sp_func_class_paren_empty = remove sp_return_paren = force @@ -136,7 +138,7 @@ sp_attribute_paren = remove sp_defined_paren = remove sp_throw_paren = force sp_after_throw = force -sp_catch_paren = force +sp_catch_paren = remove sp_macro = force sp_macro_func = force sp_else_brace = force @@ -174,8 +176,8 @@ sp_after_new = force sp_between_new_paren = force sp_after_newop_paren = force sp_inside_newop_paren = force -sp_before_tr_emb_cmt = add -sp_num_before_tr_emb_cmt = 1 +sp_before_tr_cmt = add +sp_num_before_tr_cmt = 1 sp_skip_vbrace_tokens = false sp_after_noexcept = remove @@ -205,21 +207,21 @@ indent_ctor_init_leading = 2 indent_ctor_init = 0 indent_else_if = false indent_var_def_blk = 0 -indent_var_def_cont = false -indent_shift = false +indent_var_def_cont = true +indent_shift = 0 indent_func_def_force_col1 = false -indent_func_call_param = false -indent_func_def_param = false -indent_func_proto_param = false -indent_func_class_param = false -indent_func_ctor_var_param = false -indent_template_param = false +indent_func_call_param = true +indent_func_def_param = true +indent_func_proto_param = true +indent_func_class_param = true +indent_func_ctor_var_param = true +indent_template_param = true indent_func_param_double = false indent_func_const = 0 indent_func_throw = 0 indent_member = 2 indent_member_single = true -indent_sing_line_comments = 0 +indent_single_line_comments_before = 0 indent_relative_single_line_comments = false indent_switch_case = 2 indent_switch_pp = true @@ -230,8 +232,8 @@ indent_label = 0 indent_access_spec = -2 indent_access_spec_body = false indent_paren_nl = false -indent_paren_close = 0 -indent_bool_paren = false +indent_paren_close = 2 +indent_bool_paren = 0 indent_square_nl = false indent_align_assign = true indent_align_paren = true @@ -245,7 +247,8 @@ indent_single_after_return = false indent_ignore_asm_block = false # Newline adding and removing options -nl_collapse_empty_body = false +nl_collapse_empty_body = true +nl_collapse_empty_body_functions = true nl_assign_leave_one_liners = true nl_class_leave_one_liners = true nl_enum_leave_one_liners = true @@ -261,11 +264,11 @@ nl_end_of_file = force nl_end_of_file_min = 1 nl_assign_brace = ignore nl_tsquare_brace = remove # ...or ignore -nl_func_var_def_blk = 0 +nl_var_def_blk_end_func_top = 0 nl_typedef_blk_start = 0 nl_typedef_blk_end = 0 nl_typedef_blk_in = 0 -nl_var_def_blk_start = 2 +nl_var_def_blk_start = 1 nl_var_def_blk_end = 0 nl_var_def_blk_in = 0 nl_fcall_brace = force @@ -297,7 +300,7 @@ nl_brace_while = remove # ...or ??? nl_switch_brace = force nl_synchronized_brace = force nl_multi_line_cond = false -nl_multi_line_define = false +nl_multi_line_define = true nl_before_case = false nl_after_case = false nl_case_colon_brace = force @@ -309,8 +312,8 @@ nl_template_func_def = force nl_template_func_decl = ignore nl_template_var = remove nl_class_brace = force -nl_class_init_args = ignore -nl_constr_init_args = ignore +nl_class_init_args = force +nl_constr_init_args = force nl_enum_own_lines = ignore nl_func_type_name = force nl_func_type_name_class = ignore @@ -322,8 +325,8 @@ nl_func_def_paren = remove nl_func_call_paren = remove nl_func_decl_start = ignore nl_func_def_start = ignore -nl_func_decl_start_multi_line = false -nl_func_def_start_multi_line = false +nl_func_decl_start_multi_line = true +nl_func_def_start_multi_line = true nl_func_decl_args = ignore nl_func_def_args = ignore nl_func_decl_args_multi_line = false @@ -335,7 +338,7 @@ nl_func_def_end_multi_line = false nl_func_decl_empty = remove nl_func_def_empty = remove nl_func_call_empty = remove -nl_func_call_start_multi_line = false +nl_func_call_start_multi_line = true nl_func_call_args_multi_line = false nl_func_call_end_multi_line = false nl_fdef_brace = force @@ -352,9 +355,9 @@ nl_after_vbrace_open_empty = false nl_after_brace_close = true nl_after_vbrace_close = true nl_brace_struct_var = remove -nl_define_macro = false +nl_define_macro = true nl_squeeze_paren_close = true # check me -nl_squeeze_ifdef = false +nl_squeeze_ifdef = true nl_squeeze_ifdef_top_level = false nl_ds_struct_enum_cmt = false nl_ds_struct_enum_close_brace = false @@ -414,7 +417,7 @@ pos_class_colon = lead pos_constr_colon = lead # Line Splitting options -code_width = 79 +code_width = 80 ls_for_split_full = false ls_func_split_full = false ls_code_width = false @@ -422,7 +425,7 @@ ls_code_width = false # Code alignment (not left column spaces/tabs) align_keep_tabs = false align_with_tabs = false -align_on_tabstop = true +align_on_tabstop = false align_number_right = false align_keep_extra_space = false align_func_params = false @@ -468,7 +471,7 @@ align_mix_var_proto = false align_single_line_func = false align_single_line_brace = false align_single_line_brace_gap = 0 -align_nl_cont = false +align_nl_cont = 1 align_pp_define_together = false align_pp_define_gap = 0 align_pp_define_span = 0 @@ -476,7 +479,7 @@ align_left_shift = true align_asm_colon = false # Comment modifications -cmt_width = 79 +cmt_width = 80 cmt_reflow_mode = 0 cmt_convert_tab_to_spaces = true cmt_indent_multi = true @@ -508,7 +511,9 @@ mod_add_long_switch_closebrace_comment = 0 mod_add_long_ifdef_endif_comment = 0 mod_add_long_ifdef_else_comment = 0 mod_sort_include = true +mod_remove_duplicate_include = true mod_move_case_break = true +mod_move_case_return = true mod_case_brace = ignore mod_remove_empty_return = true mod_enum_last_comma = add @@ -517,19 +522,20 @@ mod_enum_last_comma = add pp_indent = remove pp_indent_at_level = false pp_indent_count = 0 -pp_space = force -pp_space_count = 1 +pp_space_after = remove +pp_space_count = 0 pp_indent_region = 0 pp_region_indent_code = false pp_indent_if = 0 pp_if_indent_code = false pp_define_at_level = false pp_ignore_define_body = false +pp_multiline_define_body_indent = 0 # TODO next four default to true, but is that right? pp_indent_case = true pp_indent_func_def = true pp_indent_extern = true -pp_indent_brace = true +pp_indent_brace = 1 # Use or Do not Use options use_indent_func_call_param = true @@ -539,3 +545,8 @@ use_options_overriding_for_qt_macros = false # Project-specific parsing options set FUNC_WRAP TEST TEST_F TEST_P TYPED_TEST +set func_call_user LOG_TRACE +set func_call_user LOG_INFO +set func_call_user LOG_DEBUG +set func_call_user LOG_WARN +set func_call_user LOG_ERROR diff --git a/CMake/kwiver-setup-python.cmake b/CMake/kwiver-setup-python.cmake index 979fec5205..3d19c4d676 100644 --- a/CMake/kwiver-setup-python.cmake +++ b/CMake/kwiver-setup-python.cmake @@ -134,14 +134,17 @@ set(__prev_kwiver_pyversion "${KWIVER_PYTHON_MAJOR_VERSION}" CACHE INTERNAL # if (KWIVER_PYTHON_MAJOR_VERSION STREQUAL "3") # note, 3.4 is a minimum version - find_package(PythonInterp 3.4 REQUIRED) - find_package(PythonLibs 3.4 REQUIRED) + find_package(Python 3.4 COMPONENTS Interpreter Development REQUIRED) else() - find_package(PythonInterp 2.7 REQUIRED) - find_package(PythonLibs 2.7 REQUIRED) + find_package(Python 2.7 COMPONENTS Interpreter Development REQUIRED) endif() -include_directories(SYSTEM ${PYTHON_INCLUDE_DIR}) +set(PYTHON_EXECUTABLE ${Python_EXECUTABLE} CACHE FILEPATH "Path to Python executable") +set(PYTHON_INCLUDE_DIR ${Python_INCLUDE_DIRS} CACHE STRING "Paths to Python include directories") +set(PYTHON_LIBRARY ${Python_LIBRARIES} CACHE STRING "Paths to Python libraries") +set(PYTHON_LIBRARY_DEBUG PYTHON_LIBRARY_DEBUG_NOT_FOUND CACHE FILEPATH "Path to Python debug libraries") + +include_directories(SYSTEM ${PYTHON_INCLUDE_DIR}) ### # Python site-packages diff --git a/CMake/utils/kwiver-utils-tests.cmake b/CMake/utils/kwiver-utils-tests.cmake index 411a099380..e57cc54a7e 100644 --- a/CMake/utils/kwiver-utils-tests.cmake +++ b/CMake/utils/kwiver-utils-tests.cmake @@ -63,6 +63,9 @@ include(GoogleTest) +set(CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE PRE_TEST CACHE STRING + "When to run test discovery: POST_BUILD or PRE_TEST") + option(KWIVER_TEST_ADD_TARGETS "Add targets for tests to the build system" OFF) mark_as_advanced(KWIVER_TEST_ADD_TARGETS) if (KWIVER_TEST_ADD_TARGETS) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec60bbdc7b..c93d2e2163 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) # Set default visibility set(CMAKE_CXX_VISIBILITY_PRESET hidden) +# Allow capitalized _ROOT variables +if(POLICY_CMP0144) + cmake_policy(SET CMP0144 NEW) +endif() + # Organize target into folders for IDEs that support it set_property(GLOBAL PROPERTY USE_FOLDERS ON) diff --git a/Dockerfile b/Dockerfile index 065307bf1e..712e9c9b19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,66 +1,79 @@ # Install KWIVER to /opt/kitware/kwiver -# Use latest Fletch as base image (Ubuntu 18.04) +# Use latest Fletch as base image (Ubuntu 20.04) -FROM kitware/fletch:latest-ubuntu18.04-py3-cuda10.0-cudnn7-devel +ARG BASE_IMAGE=kitware/fletch:latest-ubuntu20.04-py3 +ARG ENABLE_CUDA=OFF -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3-dev \ - python3-pip \ - xvfb \ - && pip3 install setuptools \ - scipy \ - six +FROM ${BASE_IMAGE} -# -# Build KWIVER -# +# Install system dependencies +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update +RUN apt-get upgrade -y +RUN apt-get install -y --no-install-recommends xvfb +# Remove unnecessary files +RUN apt-get clean +RUN rm -rf /var/lib/apt/lists/* + +# Setup build environment COPY . /kwiver -RUN cd /kwiver \ - && mkdir build \ - && cd build \ - && cmake ../ -DCMAKE_BUILD_TYPE=Release \ - -Dfletch_DIR:PATH=/opt/kitware/fletch/share/cmake \ - -DKWIVER_ENABLE_ARROWS=ON \ - -DKWIVER_ENABLE_C_BINDINGS=ON \ - -DKWIVER_ENABLE_CERES=ON \ - -DKWIVER_ENABLE_CUDA=ON \ - -DKWIVER_ENABLE_EXTRAS=ON \ - -DKWIVER_ENABLE_LOG4CPLUS=ON \ - -DKWIVER_ENABLE_OPENCV=ON \ - -DKWIVER_ENABLE_FFMPEG=ON \ - -DKWIVER_ENABLE_KLV=ON \ - -DKWIVER_ENABLE_MVG=ON \ - -DKWIVER_ENABLE_PROCESSES=ON \ - -DKWIVER_ENABLE_PROJ=ON \ - -DKWIVER_ENABLE_PYTHON=ON \ - -DKWIVER_ENABLE_SERIALIZE_JSON=ON \ - -DKWIVER_ENABLE_SPROKIT=ON \ - -DKWIVER_ENABLE_TESTS=ON \ - -DKWIVER_ENABLE_TOOLS=ON \ - -DKWIVER_ENABLE_VXL=ON \ - -DKWIVER_ENABLE_DOCS=ON \ - -DKWIVER_INSTALL_DOCS=ON \ - -DKWIVER_PYTHON_MAJOR_VERSION=3 \ - -DKWIVER_USE_BUILD_TREE=ON \ - && make -j$(nproc) -k \ - && make install \ - && chmod +x setup_KWIVER.sh -# Optionally install python build requirements if it was generated. -RUN PYTHON_REQS="python/requirements.txt" \ - && cd /kwiver/build \ - && ( ( [ ! -f "$PYTHON_REQS" ] \ - && echo "!!! No build requirements generated, nothing to install." ) \ - || pip3 install -r "$PYTHON_REQS" ) +RUN cd /kwiver && mkdir build + +# Configure +RUN cd /kwiver/build && \ + cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/kitware/kwiver \ + -Dfletch_DIR:PATH=/opt/kitware/fletch/share/cmake \ + -DKWIVER_ENABLE_ARROWS=ON \ + -DKWIVER_ENABLE_C_BINDINGS=ON \ + -DKWIVER_ENABLE_CERES=ON \ + -DKWIVER_ENABLE_CUDA=${ENABLE_CUDA} \ + -DKWIVER_ENABLE_EXTRAS=ON \ + -DKWIVER_ENABLE_LOG4CPLUS=ON \ + -DKWIVER_ENABLE_OPENCV=ON \ + -DKWIVER_ENABLE_FFMPEG=ON \ + -DKWIVER_ENABLE_KLV=ON \ + -DKWIVER_ENABLE_MVG=ON \ + -DKWIVER_ENABLE_PROCESSES=ON \ + -DKWIVER_ENABLE_PROJ=ON \ + -DKWIVER_ENABLE_PYTHON=ON \ + -DKWIVER_ENABLE_SERIALIZE_JSON=ON \ + -DKWIVER_ENABLE_SPROKIT=ON \ + -DKWIVER_ENABLE_TESTS=ON \ + -DKWIVER_ENABLE_TOOLS=ON \ + -DKWIVER_ENABLE_VXL=ON \ + -DKWIVER_ENABLE_DOCS=ON \ + -DKWIVER_INSTALL_DOCS=ON \ + -DKWIVER_PYTHON_MAJOR_VERSION=3 \ + -DKWIVER_USE_BUILD_TREE=ON \ + -DKWIVER_INSTALL_SET_UP_SCRIPT=ON + +# Build +RUN cd /kwiver/build && \ + . ./setup_KWIVER.sh && \ + make -j$(nproc) -k && \ + make install + +# Install python build requirements if they exist +ENV PYTHON_REQS="/kwiver/build/python/requirements.txt" +RUN [ ! -f "$PYTHON_REQS" ] || pip3 install -r "$PYTHON_REQS" + +# Remove source +RUN rm -rf /kwiver # Configure entrypoint -RUN bash -c 'echo -e "source /kwiver/build/setup_KWIVER.sh\n\ -\n# Set up X virtual framebuffer (Xvfb) to support running VTK headless\n\ +RUN bash -c '\ +#!/bin/bash\n\ +echo -e "source /opt/kitware/kwiver/setup_KWIVER.sh\n\ +\n\ +# Set up X virtual framebuffer (Xvfb) to support running VTK headless\n\ Xvfb :1 -screen 0 1024x768x16 -nolisten tcp > xvfb.log &\n\ -export DISPLAY=:1.0\n\n\ -kwiver \$@" >> entrypoint.sh' \ - && chmod +x entrypoint.sh +export DISPLAY=:1.0\n\ +\n\ +/opt/kitware/kwiver/bin/kwiver \$@" > /entrypoint.sh' && \ + chmod +x /entrypoint.sh ENTRYPOINT [ "bash", "/entrypoint.sh" ] CMD [ "help" ] diff --git a/arrows/core/applets/dump_klv.cxx b/arrows/core/applets/dump_klv.cxx index a2bc5aa999..fe2a12432e 100644 --- a/arrows/core/applets/dump_klv.cxx +++ b/arrows/core/applets/dump_klv.cxx @@ -173,10 +173,13 @@ ::run() "metadata_serializer", config, metadata_serializer_ptr ); kva::metadata_map_io::get_nested_algo_configuration( "metadata_serializer", config, metadata_serializer_ptr ); - kva::image_io::set_nested_algo_configuration( - "image_writer", config, image_writer ); - kva::image_io::get_nested_algo_configuration( - "image_writer", config, image_writer ); + if( cmd_args.count( "frames" ) ) + { + kva::image_io::set_nested_algo_configuration( + "image_writer", config, image_writer ); + kva::image_io::get_nested_algo_configuration( + "image_writer", config, image_writer ); + } // Check to see if we are to dump config if ( cmd_args.count("output") ) @@ -209,7 +212,8 @@ ::run() return EXIT_FAILURE; } - if( !kva::image_io::check_nested_algo_configuration( + if( cmd_args.count( "frames" ) && + !kva::image_io::check_nested_algo_configuration( "image_writer", config ) ) { std::cerr << "Invalid image_writer config" << std::endl; @@ -223,14 +227,12 @@ ::run() } catch ( kv::video_exception const& e ) { - std::cerr << "Video Exception-Couldn't open \"" << video_file << "\"" << std::endl - << e.what() << std::endl; + std::cerr << e.what() << std::endl; return EXIT_FAILURE; } catch ( kv::file_not_found_exception const& e ) { - std::cerr << "Couldn't open \"" << video_file << "\"" << std::endl - << e.what() << std::endl; + std::cerr << e.what() << std::endl; return EXIT_FAILURE; } diff --git a/arrows/core/applets/transcode.cxx b/arrows/core/applets/transcode.cxx index 053bffb52a..103d0aa530 100644 --- a/arrows/core/applets/transcode.cxx +++ b/arrows/core/applets/transcode.cxx @@ -136,9 +136,26 @@ ::run() std::cerr << "Failed to initialize video input." << std::endl; exit( EXIT_FAILURE ); } - input->open( input_filename ); + + try + { + input->open( input_filename ); + } + catch( kv::video_runtime_exception const& e ) + { + std::cerr << e.what() << std::endl; + exit( EXIT_FAILURE ); + } + catch( kv::file_not_found_exception const& e ) + { + std::cerr << e.what() << std::endl; + exit( EXIT_FAILURE ); + } check_input( input, cmd_args ); + // Acquire first frame, which may help produce more accurate video settings + kv::timestamp timestamp; + input->next_frame( timestamp ); auto const video_settings = input->implementation_settings(); // Setup video output @@ -151,10 +168,7 @@ ::run() output->open( output_filename, video_settings.get() ); // Transcode frames - kv::timestamp timestamp; - for( input->next_frame( timestamp ); - !input->end_of_video(); - input->next_frame( timestamp ) ) + for( ; !input->end_of_video(); input->next_frame( timestamp ) ) { // Transcode metadata if( cmd_args.count( "copy-metadata" ) ) @@ -176,6 +190,13 @@ ::run() } } + // Transcode uninterpreted data + auto const misc_data = input->uninterpreted_frame_data(); + if( misc_data ) + { + output->add_uninterpreted_data( *misc_data ); + } + // Transcode image if( cmd_args.count( "copy-video" ) ) { diff --git a/arrows/core/csv_io.cxx b/arrows/core/csv_io.cxx index 1b3c3bfba5..8bd5522e4d 100644 --- a/arrows/core/csv_io.cxx +++ b/arrows/core/csv_io.cxx @@ -16,6 +16,7 @@ #include #include +#include namespace kwiver { @@ -122,6 +123,14 @@ ::write( T const& value ) { m_ss << static_cast< uint64_t >( value ); } + else if constexpr( std::is_same_v< T, float > ) + { + m_ss << std::setprecision( FLT_DIG + 1 ) << value; + } + else if constexpr( std::is_same_v< T, double > ) + { + m_ss << std::setprecision( DBL_DIG + 1 ) << value; + } else { m_ss << value; @@ -338,6 +347,52 @@ struct str_to_int64< true > return std::stoll( std::forward< Args >( args )... ); }; }; +// ---------------------------------------------------------------------------- +template< class T > +struct is_optional : std::false_type +{}; + +// ---------------------------------------------------------------------------- +template< class T > +struct is_optional< std::optional< T > > : std::true_type +{}; + +// ---------------------------------------------------------------------------- +template< class T > +constexpr bool is_optional_v = is_optional< T >::value; + +// ---------------------------------------------------------------------------- +template< class T > +struct decay_optional +{ + using type = T; +}; + +// ---------------------------------------------------------------------------- +template< class T > +struct decay_optional< std::optional< T > > +{ + using type = T; +}; + +// ---------------------------------------------------------------------------- +template< class T > +using decay_optional_t = typename decay_optional< T >::type; + +// ---------------------------------------------------------------------------- +template< class T > +T bad_parse( std::string const& s ) +{ + if constexpr( is_optional_v< T > ) + { + return std::nullopt; + } + else + { + throw csv_reader::parse_error( s, typeid( T ) ); + } +} + } // ---------------------------------------------------------------------------- @@ -374,7 +429,8 @@ ::read() if constexpr( std::is_arithmetic_v< T > || std::is_same_v< T, std::string > || - std::is_same_v< T, csv::skipf_t > ) + std::is_same_v< T, csv::skipf_t > || + is_optional_v< T > ) { if( !m_first_field && m_is.peek() == m_delim ) { @@ -384,6 +440,7 @@ ::read() m_first_field = false; auto are_in_quotes = m_is.peek() == m_quote; + auto const were_in_quotes = are_in_quotes; if( are_in_quotes ) { // Skip opening quote character @@ -501,15 +558,27 @@ ::read() return {}; } - // If all we wanted was a string, just return that + // Get unquoted field text std::string s = ss.str(); - if constexpr( std::is_same_v< T, std::string > ) + + // Check for empty field + if constexpr( is_optional_v< T > ) + { + if( s.empty() && !were_in_quotes ) + { + return std::nullopt; + } + } + using decayed_t = decay_optional_t< T >; + + // If all we wanted was a string, just return that + if constexpr( std::is_same_v< decayed_t, std::string > ) { return s; } // Convert string to boolean - if constexpr( std::is_same_v< T, bool > ) + if constexpr( std::is_same_v< decayed_t, bool > ) { if( s == "0" || s == "false" ) { @@ -520,23 +589,23 @@ ::read() return true; } - throw parse_error( s, typeid( T ) ); + return bad_parse< T >( s ); } // Numbers should not have leading whitespace - if constexpr( std::is_arithmetic_v< T > ) + if constexpr( std::is_arithmetic_v< decayed_t > ) { if( s.empty() || std::isspace( s[ 0 ] ) ) { - throw parse_error( s, typeid( T ) ); + return bad_parse< T >( s ); } } // Convert string to integer - if constexpr( std::is_integral_v< T > ) + if constexpr( std::is_integral_v< decayed_t > ) { // Parse into a 64-bit integer - using parser_t = str_to_int64< std::is_signed_v< T > >; + using parser_t = str_to_int64< std::is_signed_v< decayed_t > >; parser_t parser; typename parser_t::type value = 0; size_t offset = 0; @@ -546,15 +615,15 @@ ::read() } catch( std::exception const& ) { - throw parse_error( s, typeid( T ) ); + return bad_parse< T >( s ); } // Check if reducing to the desired integer size would overflow - if( value < std::numeric_limits< T >::lowest() || - value > std::numeric_limits< T >::max() || + if( value < std::numeric_limits< decayed_t >::lowest() || + value > std::numeric_limits< decayed_t >::max() || !offset || offset != s.size() ) { - throw parse_error( s, typeid( T ) ); + return bad_parse< T >( s ); } // Downcast to correct type @@ -563,14 +632,16 @@ ::read() // Parse to floating point // Didn't use std::is_floating_point_v because we don't handle long double - if constexpr( std::is_same_v< T, float > || std::is_same_v< T, double > ) + if constexpr( + std::is_same_v< decayed_t, float > || + std::is_same_v< decayed_t, double > ) { - T ( *parser )( char const*, char** ) = nullptr; - if constexpr( std::is_same_v< T, float > ) + decayed_t ( *parser )( char const*, char** ) = nullptr; + if constexpr( std::is_same_v< decayed_t, float > ) { parser = &std::strtof; } - if constexpr( std::is_same_v< T, double > ) + if constexpr( std::is_same_v< decayed_t, double > ) { parser = &std::strtod; } @@ -580,7 +651,7 @@ ::read() auto const value = parser( begin, &end ); if( end == begin || end != &*s.end() ) { - throw parse_error( s, typeid( T ) ); + return bad_parse< T >( s ); } return value; } @@ -709,18 +780,31 @@ ::is_at_field() template KWIVER_ALGO_CORE_EXPORT T csv_reader::read< T >(); INSTANTIATE_READ( std::string ) +INSTANTIATE_READ( std::optional< std::string > ) INSTANTIATE_READ( bool ) +INSTANTIATE_READ( std::optional< bool > ) INSTANTIATE_READ( char ) +INSTANTIATE_READ( std::optional< char > ) INSTANTIATE_READ( uint8_t ) +INSTANTIATE_READ( std::optional< uint8_t > ) INSTANTIATE_READ( uint16_t ) +INSTANTIATE_READ( std::optional< uint16_t > ) INSTANTIATE_READ( uint32_t ) +INSTANTIATE_READ( std::optional< uint32_t > ) INSTANTIATE_READ( uint64_t ) +INSTANTIATE_READ( std::optional< uint64_t > ) INSTANTIATE_READ( int8_t ) +INSTANTIATE_READ( std::optional< int8_t > ) INSTANTIATE_READ( int16_t ) +INSTANTIATE_READ( std::optional< int16_t > ) INSTANTIATE_READ( int32_t ) +INSTANTIATE_READ( std::optional< int32_t > ) INSTANTIATE_READ( int64_t ) +INSTANTIATE_READ( std::optional< int64_t > ) INSTANTIATE_READ( float ) +INSTANTIATE_READ( std::optional< float > ) INSTANTIATE_READ( double ) +INSTANTIATE_READ( std::optional< double > ) INSTANTIATE_READ( csv::skipf_t ) INSTANTIATE_READ( csv::endl_t ) INSTANTIATE_READ( csv::comment_t ) diff --git a/arrows/core/csv_io.h b/arrows/core/csv_io.h index 3e33179889..2cda353717 100644 --- a/arrows/core/csv_io.h +++ b/arrows/core/csv_io.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -290,7 +291,10 @@ class KWIVER_ALGO_CORE_EXPORT csv_reader : public csv_io_base /// csv::comment_t to begin reading the contents of a comment as regular /// fields. Otherwise, comments will be silently skipped. Passing \c /// csv::endl_t has the same effect as \c next_line(). Passing \c - /// csv::skipf_t has the same effect as \c skip_field(). + /// csv::skipf_t has the same effect as \c skip_field(). If \p T is a \c + /// std::optional, \c std::nullopt will be returned if the field is empty or + /// if parsing the field fails. Therefore, \c parse_error will not be thrown + /// if \p T is a \c std::optional. /// /// \throws parse_error If parsing from the current field to type \p T fails. /// The current field is skipped if this occurs. A string representation of diff --git a/arrows/core/metadata_map_io_csv.cxx b/arrows/core/metadata_map_io_csv.cxx index 37b96ce7e7..d54034ca07 100644 --- a/arrows/core/metadata_map_io_csv.cxx +++ b/arrows/core/metadata_map_io_csv.cxx @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -40,6 +41,10 @@ namespace core { class metadata_map_io_csv::priv { public: + std::optional< kv::metadata_value > read_csv_item( + core::csv_reader& csv_is, + kv::vital_metadata_tag tag ); + void write_csv_item( core::csv_writer& csv_os, kv::vital_metadata_tag tag, @@ -57,6 +62,33 @@ class metadata_map_io_csv::priv namespace { +constexpr auto crs = kv::SRID::lat_lon_WGS84; + +// ---------------------------------------------------------------------------- +struct read_visitor { + template< class T > + std::optional< kv::metadata_value > + operator()() const + { + if constexpr( + std::is_same_v< T, kv::geo_point > || + std::is_same_v< T, kv::geo_polygon > ) + { + throw std::logic_error( "Complex type given to csv field reader" ); + } + else + { + if( auto const value = csv_is.read< std::optional< T > >() ) + { + return *value; + } + return std::nullopt; + } + } + + core::csv_reader& csv_is; +}; + // ---------------------------------------------------------------------------- struct write_visitor { template< class T > @@ -94,45 +126,114 @@ get_column_count( std::type_info const& type ) } // ---------------------------------------------------------------------------- -// Get the special name for a particular subvalue, if it exists -std::string const* -get_special_column_name( kv::vital_metadata_tag tag, size_t index ) +struct column_id +{ + kv::vital_metadata_tag tag; + size_t index; + + bool + operator<( column_id const& other ) const + { + return std::tie( tag, index ) < std::tie( other.tag, other.index ); + } + + bool + operator==( column_id const& other ) const + { + return std::tie( tag, index ) == std::tie( other.tag, other.index ); + } +}; + +// ---------------------------------------------------------------------------- +struct special_column_name +{ + column_id id; + std::string name; +}; + +// ---------------------------------------------------------------------------- +std::vector< special_column_name > const& +special_column_names() { - static std::map< kv::vital_metadata_tag, - std::vector< std::string > > const map = { - { kv::VITAL_META_SENSOR_LOCATION, - { "Sensor Geodetic Longitude (EPSG:4326)", - "Sensor Geodetic Latitude (EPSG:4326)", - "Sensor Geodetic Altitude (meters)", } }, - { kv::VITAL_META_FRAME_CENTER, - { "Geodetic Frame Center Longitude (EPSG:4326)", - "Geodetic Frame Center Latitude (EPSG:4326)", - "Geodetic Frame Center Elevation (meters)", } }, - { kv::VITAL_META_TARGET_LOCATION, - { "Target Geodetic Location Longitude (EPSG:4326)", - "Target Geodetic Location Latitude (EPSG:4326)", - "Target Geodetic Location Elevation (meters)", } }, - { kv::VITAL_META_CORNER_POINTS, - { "Upper Left Corner Longitude (EPSG:4326)", - "Upper Left Corner Latitude (EPSG:4326)", - "Upper Right Corner Longitude (EPSG:4326)", - "Upper Right Corner Latitude (EPSG:4326)", - "Lower Right Corner Longitude (EPSG:4326)", - "Lower Right Corner Latitude (EPSG:4326)", - "Lower Left Corner Longitude (EPSG:4326)", - "Lower Left Corner Latitude (EPSG:4326)", } }, + static std::vector< special_column_name > const names = { + { { kv::VITAL_META_SENSOR_LOCATION, 0 }, + "Sensor Geodetic Longitude (EPSG:4326)" }, + { { kv::VITAL_META_SENSOR_LOCATION, 1 }, + "Sensor Geodetic Latitude (EPSG:4326)" }, + { { kv::VITAL_META_SENSOR_LOCATION, 2 }, + "Sensor Geodetic Altitude (meters)" }, + + { { kv::VITAL_META_TARGET_LOCATION, 0 }, + "Target Geodetic Location Longitude (EPSG:4326)" }, + { { kv::VITAL_META_TARGET_LOCATION, 1 }, + "Target Geodetic Location Latitude (EPSG:4326)" }, + { { kv::VITAL_META_TARGET_LOCATION, 2 }, + "Target Geodetic Location Altitude (meters)" }, + + { { kv::VITAL_META_FRAME_CENTER, 0 }, + "Geodetic Frame Center Longitude (EPSG:4326)" }, + { { kv::VITAL_META_FRAME_CENTER, 1 }, + "Geodetic Frame Center Longitude (EPSG:4326)" }, + { { kv::VITAL_META_FRAME_CENTER, 2 }, + "Geodetic Frame Center Altitude (meters)" }, + + { { kv::VITAL_META_CORNER_POINTS, 0 }, + "Upper Left Corner Longitude (EPSG:4326)" }, + { { kv::VITAL_META_CORNER_POINTS, 1 }, + "Upper Left Corner Latitude (EPSG:4326)" }, + { { kv::VITAL_META_CORNER_POINTS, 2 }, + "Upper Right Corner Longitude (EPSG:4326)" }, + { { kv::VITAL_META_CORNER_POINTS, 3 }, + "Upper Right Corner Latitude (EPSG:4326)" }, + { { kv::VITAL_META_CORNER_POINTS, 4 }, + "Lower Right Corner Longitude (EPSG:4326)" }, + { { kv::VITAL_META_CORNER_POINTS, 5 }, + "Lower Right Corner Latitude (EPSG:4326)" }, + { { kv::VITAL_META_CORNER_POINTS, 6 }, + "Lower Left Corner Longitude (EPSG:4326)" }, + { { kv::VITAL_META_CORNER_POINTS, 7 }, + "Lower Left Corner Latitude (EPSG:4326)" }, }; - auto const it = map.find( tag ); - return ( it != map.end() ) ? &it->second.at( index ) : nullptr; + return names; +} + +// ---------------------------------------------------------------------------- +// Get the special name for a particular subvalue, if it exists. +std::string const* +get_special_column_name( column_id const& id ) +{ + for( auto const& entry : special_column_names() ) + { + if( entry.id == id ) + { + return &entry.name; + } + } + return nullptr; +} + +// ---------------------------------------------------------------------------- +// Get the subvalue for a particular special column name, if it exists. +column_id const* +get_special_column_id( std::string const& name ) +{ + for( auto const& entry : special_column_names() ) + { + if( entry.name == name ) + { + return &entry.id; + } + } + return nullptr; } // ---------------------------------------------------------------------------- // Get the name to be used as the header title for the given subvalue. std::string -get_column_name( kv::vital_metadata_tag tag, size_t index, bool use_enum_name ) +get_column_name( column_id const& id, bool use_enum_name ) { - auto const& traits = kv::tag_traits_by_tag( tag ); + auto const& traits = kv::tag_traits_by_tag( id.tag ); auto const column_count = get_column_count( traits.type() ); std::stringstream ss; if( use_enum_name ) @@ -140,12 +241,12 @@ get_column_name( kv::vital_metadata_tag tag, size_t index, bool use_enum_name ) ss << traits.enum_name(); if( column_count > 1 ) { - ss << '.' << index; + ss << '.' << id.index; } } else { - auto const special_name = get_special_column_name( tag, index ); + auto const special_name = get_special_column_name( id ); if( special_name ) { ss << *special_name; @@ -155,7 +256,7 @@ get_column_name( kv::vital_metadata_tag tag, size_t index, bool use_enum_name ) ss << traits.name(); if( column_count > 1 ) { - ss << '.' << index; + ss << '.' << id.index; } } } @@ -163,6 +264,41 @@ get_column_name( kv::vital_metadata_tag tag, size_t index, bool use_enum_name ) return ss.str(); } +// ---------------------------------------------------------------------------- +// Determine what subvalue is being requested via the given string. +column_id +parse_column_id( std::string const& s ) +{ + if( auto const special_id = get_special_column_id( s ) ) + { + return *special_id; + } + + // Format of s will be: NAME.INDEX or just NAME (index defaults to 0) + // NAME will be either enum_name or regular name of a vital tag + column_id result = { kv::VITAL_META_UNKNOWN, 0 }; + auto const separator_pos = s.rfind( '.' ); + auto name = s; + if( separator_pos != s.npos ) + { + try + { + result.index = std::stoi( s.substr( separator_pos + 1 ) ); + name = s.substr( 0, separator_pos ); + } + catch( std::invalid_argument const& e ) + { + // Maybe there was a period in the name? + } + } + if( ( result.tag = kv::tag_traits_by_enum_name( name ).tag() ) == + kv::VITAL_META_UNKNOWN ) + { + result.tag = kv::tag_traits_by_name( name ).tag(); + } + return result; +} + // ---------------------------------------------------------------------------- struct subvalue_visitor { @@ -182,7 +318,7 @@ kv::metadata_value subvalue_visitor ::operator()< kv::geo_point >( kv::geo_point const& value ) const { - return value.location( kv::SRID::lat_lon_WGS84 )( index ); + return value.location( crs )( index ); } // ---------------------------------------------------------------------------- @@ -191,7 +327,7 @@ kv::metadata_value subvalue_visitor ::operator()< kv::geo_polygon >( kv::geo_polygon const& value ) const { - return value.polygon( kv::SRID::lat_lon_WGS84 ).at( index / 2 )( index % 2 ); + return value.polygon( crs ).at( index / 2 )( index % 2 ); } // ---------------------------------------------------------------------------- @@ -203,50 +339,95 @@ get_subvalue( kv::metadata_value const& value, size_t index ) } // ---------------------------------------------------------------------------- -struct column_id +struct set_subvalue_visitor { - kv::vital_metadata_tag tag; - size_t index; - - bool - operator<( column_id const& other ) const + template< class T > + void + operator()() const { - if( tag < other.tag ) { return true; } - if( tag > other.tag ) { return false; } - return index < other.index; + constexpr auto nan = std::numeric_limits< double >::quiet_NaN(); + if constexpr( std::is_same_v< T, kv::geo_point > ) + { + static T const default_value{ kv::vector_3d{ nan, nan, nan }, crs }; + auto original_value = + metadata.has( column.tag ) + ? metadata.find( column.tag ).get< T >() + : default_value; + auto internal_value = original_value.location( crs ); + internal_value( column.index ) = std::get< double >( value ); + original_value.set_location( internal_value, crs ); + metadata.add( column.tag, original_value ); + } + else if constexpr( std::is_same_v< T, kv::geo_polygon > ) + { + static T const default_value{ + std::vector( 4, kv::vector_2d{ nan, nan } ), crs }; + auto original_value = + metadata.has( column.tag ) + ? metadata.find( column.tag ).get< T >() + : default_value; + auto internal_value = original_value.polygon( crs ).get_vertices(); + internal_value.at( column.index / 2 )( column.index % 2 ) = + std::get< double >( value ); + original_value.set_polygon( internal_value, crs ); + metadata.add( column.tag, original_value ); + } + else + { + metadata.add( column.tag, value ); + } } + + column_id const& column; + kv::metadata_value const& value; + kv::metadata& metadata; }; +} // namespace + // ---------------------------------------------------------------------------- -// Determine what subvalue is being requested via the given string. -column_id -parse_column_id( std::string const& s ) +std::optional< kv::metadata_value > +metadata_map_io_csv::priv +::read_csv_item( core::csv_reader& csv_is, kv::vital_metadata_tag tag ) { - // Format of s will be: NAME.INDEX or just NAME (index defaults to 0) - // NAME will be either enum_name or regular name of a vital tag - column_id result = { kv::VITAL_META_UNKNOWN, 0 }; - auto const separator_pos = s.rfind( '.' ); - auto name = s; - if( separator_pos != s.npos ) + if( tag == kv::VITAL_META_VIDEO_MICROSECONDS ) { - try + auto const maybe_s = csv_is.read< std::optional< std::string > >(); + if( !maybe_s ) { - result.index = std::stoi( s.substr( separator_pos + 1 ) ); - name = s.substr( 0, separator_pos ); + return std::nullopt; } - catch( std::invalid_argument const& e ) + auto const& s = *maybe_s; + + static std::regex const pattern( "(\\d{2}):(\\d{2}):(\\d{2}).(\\d{6})" ); + std::smatch match; + if( !std::regex_match( s, match, pattern ) ) { - // Maybe there was a period in the name? + return std::nullopt; } + uint64_t microseconds = 0; + auto const convert = + [ &match, µseconds ]( size_t index, uint64_t factor ) { + microseconds *= factor; + microseconds += std::stoull( match.str( index ) ); + }; + convert( 1, 0 ); + convert( 2, 60 ); + convert( 3, 60 ); + convert( 4, 1000000 ); + return microseconds; } - if( ( result.tag = kv::tag_traits_by_enum_name( name ).tag() ) == - kv::VITAL_META_UNKNOWN ) + else { - result.tag = kv::tag_traits_by_name( name ).tag(); - } - return result; -} + auto const* type = &kv::tag_traits_by_tag( tag ).type(); + if( *type == typeid( kv::geo_point ) || *type == typeid( kv::geo_polygon ) ) + { + type = &typeid( double ); + } + return kv::visit_metadata_types_return< + std::optional< kv::metadata_value >, read_visitor >( { csv_is }, *type ); + } } // ---------------------------------------------------------------------------- @@ -373,9 +554,75 @@ ::get_configuration() const // ---------------------------------------------------------------------------- kv::metadata_map_sptr metadata_map_io_csv -::load_( VITAL_UNUSED std::istream& fin, std::string const& filename ) const +::load_( std::istream& is, std::string const& filename ) const { - throw kv::file_write_exception( filename, "not implemented" ); + // Check that output file is valid + if( !is ) + { + VITAL_THROW( + kv::invalid_file, filename, "Insufficient permissions or moved file" ); + } + + // Initialize reader + core::csv_reader csv_is{ is }; + + // Parse column names + std::vector< column_id > column_ids{ + { kv::VITAL_META_VIDEO_FRAME_NUMBER, 0 } }; + if( csv_is.read< std::string >() != "Frame ID" ) + { + VITAL_THROW( + kv::invalid_file, filename, "First column must be 'Frame ID'" ); + } + while( !csv_is.is_at_eol() ) + { + auto const name = csv_is.read< std::string >(); + column_ids.emplace_back( parse_column_id( name ) ); + } + + // Parse remaining lines + vital::simple_metadata_map::map_metadata_t result; + while( !csv_is.is_at_eof() ) + { + csv_is.next_line(); + + // Parse each column in turn + std::map< column_id, kv::metadata_value > values; + for( auto const& column : column_ids ) + { + if( auto const value = d_->read_csv_item( csv_is, column.tag ) ) + { + if( !values.emplace( column, *value ).second ) + { + LOG_WARN( logger(), + "Dropping duplicate value for column: " + << get_column_name( column, true ) ); + } + } + } + + // Create an empty metadata packet for this frame + using frame_number_t = + vital::type_of_tag< vital::VITAL_META_VIDEO_FRAME_NUMBER >; + auto const& frame_number_value = + values.at( { vital::VITAL_META_VIDEO_FRAME_NUMBER, 0 } ); + auto const frame_number = std::get< frame_number_t >( frame_number_value ); + auto& metadata = + *result.emplace( frame_number, kv::metadata_vector{} ).first->second + .emplace_back( std::make_shared< kv::metadata >() ); + + // Fill that metadata packet with the values, correctly handling + // multi-column fields + for( auto const& entry : values ) + { + auto const& tag_type = + vital::tag_traits_by_tag( entry.first.tag ).type(); + kv::visit_metadata_types< set_subvalue_visitor >( + { entry.first, entry.second, metadata }, tag_type ); + } + } + + return std::make_shared< kv::simple_metadata_map >( result ); } // ---------------------------------------------------------------------------- @@ -430,7 +677,7 @@ ::save_( std::ostream& os, info.id = parse_column_id( name ); info.name = name_override.empty() - ? get_column_name( info.id.tag, info.id.index, d_->write_enum_names ) + ? get_column_name( info.id, d_->write_enum_names ) : name_override; if( info.id.tag != kv::VITAL_META_UNKNOWN ) @@ -447,8 +694,7 @@ ::save_( std::ostream& os, { for( auto const& id : remaining_ids ) { - auto const name = - get_column_name( id.tag, id.index, d_->write_enum_names ); + auto const name = get_column_name( id, d_->write_enum_names ); infos.push_back( { id, name } ); } } diff --git a/arrows/core/tests/CMakeLists.txt b/arrows/core/tests/CMakeLists.txt index 419679b9ab..95da52ebf9 100644 --- a/arrows/core/tests/CMakeLists.txt +++ b/arrows/core/tests/CMakeLists.txt @@ -32,3 +32,4 @@ kwiver_discover_gtests(core video_input_image_list LIBRARIES ${test_libraries kwiver_discover_gtests(core video_input_pos LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") kwiver_discover_gtests(core video_input_splice LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") kwiver_discover_gtests(core video_input_split LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") +kwiver_discover_gtests(core match_matrix LIBRARIES ${test_libraries}) \ No newline at end of file diff --git a/arrows/core/tests/test_csv_io.cxx b/arrows/core/tests/test_csv_io.cxx index a240efb8ea..d52f807833 100644 --- a/arrows/core/tests/test_csv_io.cxx +++ b/arrows/core/tests/test_csv_io.cxx @@ -377,3 +377,30 @@ TEST ( csv_io, read_comment ) reader.skip_line(); EXPECT_TRUE( reader.is_at_eof() ); } + +// ---------------------------------------------------------------------------- +TEST ( csv_io, read_optional ) +{ + std::stringstream ss{ ",,,,,A,Maybe,1L,-10,five,0,true,\"\",5,7.0" }; + csv_reader reader( ss ); + + EXPECT_EQ( std::nullopt, reader.read< std::optional< char > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< bool > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< std::string > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< uint16_t > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< float > >() ); + + EXPECT_EQ( std::nullopt, reader.read< std::optional< char > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< bool > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< uint16_t > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< uint16_t > >() ); + EXPECT_EQ( std::nullopt, reader.read< std::optional< float > >() ); + + EXPECT_EQ( 0, reader.read< std::optional< char > >() ); + EXPECT_EQ( true, reader.read< std::optional< bool > >() ); + EXPECT_EQ( std::string{}, reader.read< std::optional< std::string > >() ); + EXPECT_EQ( 5, reader.read< std::optional< uint16_t > >() ); + EXPECT_EQ( 7.0f, reader.read< std::optional< float > >() ); + + EXPECT_TRUE( reader.is_at_eol() ); +} diff --git a/arrows/core/tests/test_match_matrix.cxx b/arrows/core/tests/test_match_matrix.cxx new file mode 100644 index 0000000000..1293708f85 --- /dev/null +++ b/arrows/core/tests/test_match_matrix.cxx @@ -0,0 +1,376 @@ +/*ckwg +29 + * Copyright 2014-2017 by Kitware, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither name of Kitware, Inc. nor the names of any contributors may be used + * to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include + +using namespace kwiver::vital; + +// This can be removed prior to merging with source code +void view_set_matrix(); + +// ---------------------------------------------------------------------------- +int main( int argc, char** argv ) +{ + // This can be removed prior to merging with source code + view_set_matrix(); + + ::testing::InitGoogleTest( &argc, argv ); + return RUN_ALL_TESTS(); +} + +namespace { + +// ---------------------------------------------------------------------------- +// Helper function to generate deterministic track set +kwiver::vital::track_set_sptr +gen_set_tracks( unsigned frames=100, + unsigned max_tracks_per_frame=1000, + unsigned min_tracks_per_frame=500, + double termination_fraction = 0.01, + double skip_fraction = 0.01, + double frame_drop_fraction = 0.01 ) +{ + + // Manually terminate tracks on frames 0, 1 and 3 + track_id_t track_id=0; + std::vector< track_sptr > all_tracks, active_tracks; + for( unsigned f=0; fset_id(track_id++); + active_tracks.push_back(t); + all_tracks.push_back(t); + } + + // Add a state for each track to this frame + for( auto t : active_tracks ) + { + t->append( std::make_shared( f ) ); + } + + if(f==0) + { + // Terminate tracks 0 and 3 on frame 0 + std::vector< track_sptr > next_tracks; + for( auto t : active_tracks ) + { + if( t->id() != 0 && t->id() != 3 ) + { + next_tracks.push_back( t ); + } + } + active_tracks.swap( next_tracks ); + } + + if(f==1) + { + // Terminate tracks 2 and 7 on frame 1 + std::vector< track_sptr > next_tracks; + for( auto t : active_tracks ) + { + if( t->id() != 2 && t->id() != 7 ) + { + next_tracks.push_back( t ); + } + } + active_tracks.swap( next_tracks ); + } + + if(f==3) + { + // Terminate tracks 5 and 9 on frame 3 + std::vector< track_sptr > next_tracks; + for( auto t : active_tracks ) + { + if( t->id() != 5 && t->id() != 9 ) + { + next_tracks.push_back( t ); + } + } + active_tracks.swap( next_tracks ); + } + } + return std::make_shared( all_tracks ); +} + +// ---------------------------------------------------------------------------- +// Function to generate match matrix with known values +Eigen::SparseMatrix gen_test_matrix() +{ + Eigen::Matrix dense_matrix; + + // Manually calculated matrix from gen_set_tracks() + dense_matrix << 8, 6, 4, 4, 3, + 6, 8, 6, 6, 4, + 4, 6, 8, 8, 6, + 4, 6, 8, 8, 6, + 3, 4, 6, 6, 8; + + // Convert the dense matrix to a sparse matrix for unit test comparison + Eigen::SparseMatrix test_matrix = dense_matrix.sparseView(); + + return test_matrix; +} + +// ---------------------------------------------------------------------------- +// Function to calculate the max possible importance score +double gen_max_score(Eigen::SparseMatrix matrix) +{ + double sum = 0.0; + + for (int row = 0; row < matrix.rows(); ++row) + { + for (int col = 0; col <= row; ++col) { + unsigned int value = matrix.coeff(row, col); + if (value != 0) { + sum += 1.0 / static_cast(value); + } + } + } + return sum; +} + +// ---------------------------------------------------------------------------- +// Function to generate importance scores from known values for comparison +std::vector gen_set_scores() +{ + std::vector set_scores; + + // Manually calculated for the 'set_tracks' and 'set_matrix' + set_scores = {1.0/8, 8.0/3, 5.0/12, 1.0/8, 8.0/3, 1.625, 8.0/3, + 5.0/12, 37.0/24, 5.0/6, 5.0/6, 5.0/6, 1.0/8, 1.0/8}; + + return set_scores; +} + +// ---------------------------------------------------------------------------- +// Function to check range of elements in match matrix +bool matrix_values(const Eigen::SparseMatrix& matrix, + unsigned int max_tracks) +{ + for (int i = 0; i < matrix.rows(); ++i) { + for (int k = 0; k < matrix.cols(); ++k) { + unsigned int value = matrix.coeff(i, k); + if (value < 0 || value > max_tracks) { + return false; + } + } + } + return true; +} + +// ---------------------------------------------------------------------------- +// Establish constants and create variables for test_tracks + +// These parameters can be varied for further testing +const unsigned int num_frames = 100; +const unsigned int max_tracks = 1000; + +track_set_sptr test_tracks = + kwiver::testing::generate_tracks(num_frames, max_tracks); + +const auto trks = test_tracks->tracks(); + +std::set frame_ids = test_tracks->all_frame_ids(); +std::vector frames = + std::vector(frame_ids.begin(), frame_ids.end()); + +// Frames might dropped in track set generation +int actual_num_frames = test_tracks->all_frame_ids().size(); + +Eigen::SparseMatrix matched_matrix = + kwiver::arrows::match_matrix(test_tracks, frames); + +// ---------------------------------------------------------------------------- +// Establish constants and create variables for set_tracks + +// DO NOT EDIT these two constants, might cause unit tests to fail +const unsigned int set_num_frames = 5; +const unsigned int set_max_tracks = 8; + +track_set_sptr set_tracks = + gen_set_tracks(set_num_frames, set_max_tracks); + +const auto set_trks = set_tracks->tracks(); + +std::set set_frame_ids = set_tracks->all_frame_ids(); +std::vector set_frames = + std::vector(set_frame_ids.begin(), set_frame_ids.end()); + +Eigen::SparseMatrix set_matrix = + kwiver::arrows::match_matrix(set_tracks, set_frames); + +std::map set_importance_scores = + kwiver::arrows::match_matrix_track_importance(set_tracks, + set_frames, set_matrix); + +} // end namespace anonymous + +// ---------------------------------------------------------------------------- +TEST(match_matrix, matrix_dimensions) +{ + int num_rows = matched_matrix.rows(); + int num_cols = matched_matrix.cols(); + + ASSERT_EQ(num_rows, actual_num_frames); + ASSERT_EQ(num_cols, actual_num_frames); +} + +// ---------------------------------------------------------------------------- +// Test range of matrix values and symmetry +TEST(match_matrix, matrix_values) +{ + EXPECT_TRUE(matrix_values(matched_matrix, max_tracks)); + EXPECT_TRUE(matched_matrix.isApprox(matched_matrix.transpose())); +} + +// ---------------------------------------------------------------------------- +// Test matrix diagonal values match the number of tracks in each frame +TEST(match_matrix, diagonal_values) +{ + std::vector tracks_in_frame(actual_num_frames, 0); + + for (const auto& t : trks) { + std::set t_frames = t->all_frame_ids(); + for (const auto& fid : t_frames) { + tracks_in_frame[fid]++; + } + } + + std::vector diag_elements(actual_num_frames, 0); + + for (Eigen::Index i = 0; i < matched_matrix.rows(); ++i) { + diag_elements[i] =(matched_matrix.coeff(i, i)); + } + + EXPECT_EQ(diag_elements, tracks_in_frame); +} + +// ---------------------------------------------------------------------------- +// Test that match_matrix() function is equivalent to calculated matrix +TEST(match_matrix, test_matrix) +{ + Eigen::SparseMatrix test_matrix = gen_test_matrix(); + + ASSERT_TRUE(set_matrix.isApprox(test_matrix)); +} + +// ---------------------------------------------------------------------------- +TEST(importance_score, vector_size) +{ + std::map importance_scores = + kwiver::arrows::match_matrix_track_importance(test_tracks, + frames, matched_matrix); + + double max_score = gen_max_score(matched_matrix); + + std::vector score_values; + for (const auto& entry : importance_scores) + { + score_values.push_back(entry.second); + } + + double largest_score = *std::max_element(score_values.begin(), + score_values.end()); + + EXPECT_EQ(test_tracks->size(), importance_scores.size()); + EXPECT_LE(largest_score, max_score); +} + +// ---------------------------------------------------------------------------- +// Test importance score function against pre-determined result +TEST(importance_score, score_values) +{ + // invoke the importance scores that were manually calculated + auto set_scores = gen_set_scores(); + + std::vector score_values; + for (const auto& entry : set_importance_scores) + { + score_values.push_back(entry.second); + } + + const double tolerance = 1e-5; + + for (size_t i = 0; i < set_scores.size(); ++i) + { + EXPECT_NEAR(set_scores[i], score_values[i], tolerance); + } +} + +// ---------------------------------------------------------------------------- +// Function to view results for a small track set +// Used for visual inspection, manual calculations and de-bugging +// Can be removed before merging with source code +void view_set_matrix() +{ + std::cout << "Deterministic track set"<all_frame_ids().size(); + + // View each frame and associated tracks + for (frame_id_t f_id = 0; f_id < actual_set_num_frames; ++f_id) { + + std::cout << "Frame " << f_id << " - Tracks: "; + + for (const auto& t : set_trks) { + // Get all frames covered by this track + std::set t_frames = t->all_frame_ids(); + + // Check if the desired frame is in the set of + // frames covered by the track + if (t_frames.find(f_id) != t_frames.end()) { + std::cout << t->id() << " "; + } + } + + std::cout << std::endl; + } + + std::cout << '\n'; + + std::cout << "Deterministic matched matrix\n" << set_matrix << std::endl; + + std::cout << "Track Importance Scores:\n"; + for (const auto& entry : set_importance_scores) { + std::cout <<"Track ID: "<() } }, - { 7, { std::make_shared< kv::metadata >(), - std::make_shared< kv::metadata >() } } }; - - map[ 4 ][ 0 ]->add< kv::VITAL_META_UNIX_TIMESTAMP >( 1 ); - map[ 4 ][ 0 ]->add< kv::VITAL_META_VIDEO_DATA_STREAM_INDEX >( 1 ); - map[ 4 ][ 0 ]->add< kv::VITAL_META_SENSOR_HORIZONTAL_FOV >( 60.7 ); - map[ 4 ][ 0 ]->add< kv::VITAL_META_PLATFORM_DESIGNATION >( "\"Platform,\"" ); - map[ 4 ][ 0 ]->add< kv::VITAL_META_SENSOR_LOCATION >( - { kv::vector_2d{ 2.0, 3.0 }, kv::SRID::lat_lon_WGS84 } ); - map[ 4 ][ 0 ]->add< kv::VITAL_META_CORNER_POINTS >( - { { kv::vector_2d{ 0.0, 3.0 }, - kv::vector_2d{ 2.0, 3.0 }, - kv::vector_2d{ 2.0, 6.0 }, - kv::vector_2d{ 0.0, 6.0 } }, kv::SRID::lat_lon_WGS84 } ); - - map[ 7 ][ 0 ]->add< kv::VITAL_META_UNIX_TIMESTAMP >( 3 ); - map[ 7 ][ 0 ]->add< kv::VITAL_META_VIDEO_DATA_STREAM_INDEX >( 1 ); - - map[ 7 ][ 1 ]->add< kv::VITAL_META_UNIX_TIMESTAMP >( 5 ); - map[ 7 ][ 1 ]->add< kv::VITAL_META_VIDEO_DATA_STREAM_INDEX >( 2 ); - - // Create CSV writer - metadata_map_io_csv io; +protected: + void SetUp() override + { + // Set write_enum_names so this test still works if the field names change + auto const config = io.get_configuration(); + config->set_value( "write_enum_names", true ); + io.set_configuration( config ); - // Set write_enum_names so this test still works if the field names change - auto const config = io.get_configuration(); - config->set_value( "write_enum_names", true ); - io.set_configuration( config ); + map = { + { 4, { std::make_shared< kv::metadata >() } }, + { 7, { std::make_shared< kv::metadata >(), + std::make_shared< kv::metadata >() } } }; - // Write to CSV + map[ 4 ][ 0 ]->add< kv::VITAL_META_VIDEO_FRAME_NUMBER >( 4 ); + map[ 4 ][ 0 ]->add< kv::VITAL_META_UNIX_TIMESTAMP >( 1 ); + map[ 4 ][ 0 ]->add< kv::VITAL_META_VIDEO_DATA_STREAM_INDEX >( 1 ); + map[ 4 ][ 0 ]->add< kv::VITAL_META_SENSOR_HORIZONTAL_FOV >( 60.7 ); + map[ 4 ][ 0 ]->add< kv::VITAL_META_PLATFORM_DESIGNATION >( "\"Platform,\"" ); + map[ 4 ][ 0 ]->add< kv::VITAL_META_SENSOR_LOCATION >( + { kv::vector_2d{ 2.0, 3.0 }, kv::SRID::lat_lon_WGS84 } ); + map[ 4 ][ 0 ]->add< kv::VITAL_META_CORNER_POINTS >( + { { kv::vector_2d{ 0.0, 3.0 }, + kv::vector_2d{ 2.0, 3.0 }, + kv::vector_2d{ 2.0, 6.0 }, + kv::vector_2d{ 0.0, 6.0 } }, kv::SRID::lat_lon_WGS84 } ); + + map[ 7 ][ 0 ]->add< kv::VITAL_META_VIDEO_FRAME_NUMBER >( 7 ); + map[ 7 ][ 0 ]->add< kv::VITAL_META_UNIX_TIMESTAMP >( 3 ); + map[ 7 ][ 0 ]->add< kv::VITAL_META_VIDEO_MICROSECONDS >( 123456789012 ); + map[ 7 ][ 0 ]->add< kv::VITAL_META_VIDEO_DATA_STREAM_INDEX >( 1 ); + + map[ 7 ][ 1 ]->add< kv::VITAL_META_VIDEO_FRAME_NUMBER >( 7 ); + map[ 7 ][ 1 ]->add< kv::VITAL_META_UNIX_TIMESTAMP >( 5 ); + map[ 7 ][ 1 ]->add< kv::VITAL_META_VIDEO_DATA_STREAM_INDEX >( 2 ); + + example_csv = + "Frame ID,UNIX_TIMESTAMP,PLATFORM_DESIGNATION,VIDEO_DATA_STREAM_INDEX," + "VIDEO_MICROSECONDS," + "SENSOR_LOCATION.0,SENSOR_LOCATION.1,SENSOR_LOCATION.2," + "SENSOR_HORIZONTAL_FOV," + "CORNER_POINTS.0,CORNER_POINTS.1,CORNER_POINTS.2,CORNER_POINTS.3," + "CORNER_POINTS.4,CORNER_POINTS.5,CORNER_POINTS.6,CORNER_POINTS.7\n" + "4,1,\"\"\"Platform,\"\"\",1,,2,3,0,60.7,0,3,2,3,2,6,0,6\n" + "7,3,,1,34:17:36.789012,,,,,,,,,,,,\n" + "7,5,,2,,,,,,,,,,,,,\n"; + } + + metadata_map_io_csv io; + kv::metadata_map::map_metadata_t map; std::stringstream ss; + std::string example_csv; +}; + +// ---------------------------------------------------------------------------- +TEST_F( metadata_map_csv, save ) +{ + // Write to CSV io.save_( ss, std::make_shared< kv::simple_metadata_map >( map ), "" ); - std::string const expected_result = - "Frame ID,UNIX_TIMESTAMP,PLATFORM_DESIGNATION,VIDEO_DATA_STREAM_INDEX," - "SENSOR_LOCATION.0,SENSOR_LOCATION.1,SENSOR_LOCATION.2," - "SENSOR_HORIZONTAL_FOV," - "CORNER_POINTS.0,CORNER_POINTS.1,CORNER_POINTS.2,CORNER_POINTS.3," - "CORNER_POINTS.4,CORNER_POINTS.5,CORNER_POINTS.6,CORNER_POINTS.7\n" - "4,1,\"\"\"Platform,\"\"\",1,2,3,0,60.7,0,3,2,3,2,6,0,6\n" - "7,3,,1,,,,,,,,,,,,\n" - "7,5,,2,,,,,,,,,,,,\n"; - EXPECT_EQ( expected_result, ss.str() ); + EXPECT_EQ( example_csv, ss.str() ); +} + +// ---------------------------------------------------------------------------- +TEST_F( metadata_map_csv, load ) +{ + // Read from CSV + ss.str( example_csv ); + auto const result_map = io.load_( ss, "" )->metadata(); + + auto true_it = map.cbegin(); + auto result_it = result_map.cbegin(); + while( true_it != map.cend() && result_it != result_map.cend() ) + { + EXPECT_EQ( true_it->first, result_it->first ); + EXPECT_TRUE( std::equal( + true_it->second.begin(), true_it->second.end(), + result_it->second.begin(), result_it->second.end(), + []( auto const& lhs, auto const& rhs ){ return *lhs == *rhs; } ) ) + << "Frame " << true_it->first << " not equal"; + ++true_it; + ++result_it; + } + + EXPECT_EQ( true_it, map.cend() ); + EXPECT_EQ( result_it, result_map.cend() ); } diff --git a/arrows/core/video_input_buffered_metadata_filter.cxx b/arrows/core/video_input_buffered_metadata_filter.cxx index 91b331e6b2..fce9cd4562 100644 --- a/arrows/core/video_input_buffered_metadata_filter.cxx +++ b/arrows/core/video_input_buffered_metadata_filter.cxx @@ -35,6 +35,7 @@ class video_input_buffered_metadata_filter::impl kv::timestamp timestamp; kv::image_container_sptr image; kv::video_raw_image_sptr raw_image; + kv::video_uninterpreted_data_sptr uninterpreted_data; }; kv::algo::video_input_sptr video_input; @@ -50,7 +51,8 @@ video_input_buffered_metadata_filter::impl::frame_info ::frame_info( kv::algo::video_input& input ) : timestamp{ input.frame_timestamp() }, image{ input.frame_image() }, - raw_image{ input.raw_frame_image() } + raw_image{ input.raw_frame_image() }, + uninterpreted_data{ input.uninterpreted_frame_data() } {} // ---------------------------------------------------------------------------- @@ -148,7 +150,8 @@ ::open( std::string name ) vi::HAS_ABSOLUTE_FRAME_TIME, vi::HAS_TIMEOUT, vi::HAS_RAW_IMAGE, - vi::HAS_RAW_METADATA, } ) + vi::HAS_RAW_METADATA, + vi::HAS_UNINTERPRETED_DATA, } ) { set_capability( capability, capabilities.capability( capability ) ); } @@ -345,6 +348,19 @@ ::frame_metadata() return d->frame_metadata; } +// ---------------------------------------------------------------------------- +vital::video_uninterpreted_data_sptr +video_input_buffered_metadata_filter +::uninterpreted_frame_data() +{ + if( end_of_video() || d->frames.empty() ) + { + return nullptr; + } + + return d->frames.front().uninterpreted_data; +} + // ---------------------------------------------------------------------------- kv::metadata_map_sptr video_input_buffered_metadata_filter diff --git a/arrows/core/video_input_buffered_metadata_filter.h b/arrows/core/video_input_buffered_metadata_filter.h index ef4ed365b4..43c0a0d355 100644 --- a/arrows/core/video_input_buffered_metadata_filter.h +++ b/arrows/core/video_input_buffered_metadata_filter.h @@ -57,6 +57,7 @@ class KWIVER_ALGO_CORE_EXPORT video_input_buffered_metadata_filter vital::image_container_sptr frame_image() override; vital::video_raw_image_sptr raw_frame_image() override; vital::metadata_vector frame_metadata() override; + vital::video_uninterpreted_data_sptr uninterpreted_frame_data() override; vital::metadata_map_sptr metadata_map() override; vital::video_settings_uptr implementation_settings() const override; diff --git a/arrows/core/video_input_metadata_filter.cxx b/arrows/core/video_input_metadata_filter.cxx index f60b649d79..aba4fa623b 100644 --- a/arrows/core/video_input_metadata_filter.cxx +++ b/arrows/core/video_input_metadata_filter.cxx @@ -164,6 +164,7 @@ ::open( std::string name ) copy_capability( vi::IS_SEEKABLE ); copy_capability( vi::HAS_RAW_IMAGE ); copy_capability( vi::HAS_RAW_METADATA ); + copy_capability( vi::HAS_UNINTERPRETED_DATA ); } // ---------------------------------------------------------------------------- @@ -244,6 +245,19 @@ ::raw_frame_image() return m_d->video_input->raw_frame_image(); } +// ---------------------------------------------------------------------------- +kv::video_uninterpreted_data_sptr +video_input_metadata_filter +::uninterpreted_frame_data() +{ + if( !m_d->video_input ) + { + return nullptr; + } + + return m_d->video_input->uninterpreted_frame_data(); +} + // ---------------------------------------------------------------------------- kv::metadata_map_sptr video_input_metadata_filter diff --git a/arrows/core/video_input_metadata_filter.h b/arrows/core/video_input_metadata_filter.h index fb07e32254..ade51e898a 100644 --- a/arrows/core/video_input_metadata_filter.h +++ b/arrows/core/video_input_metadata_filter.h @@ -53,6 +53,7 @@ class KWIVER_ALGO_CORE_EXPORT video_input_metadata_filter kwiver::vital::image_container_sptr frame_image() override; kwiver::vital::video_raw_image_sptr raw_frame_image() override; kwiver::vital::metadata_vector frame_metadata() override; + kwiver::vital::video_uninterpreted_data_sptr uninterpreted_frame_data() override; kwiver::vital::metadata_map_sptr metadata_map() override; kwiver::vital::video_settings_uptr implementation_settings() const override; diff --git a/arrows/ffmpeg/CMakeLists.txt b/arrows/ffmpeg/CMakeLists.txt index e72f848869..5428e102dc 100644 --- a/arrows/ffmpeg/CMakeLists.txt +++ b/arrows/ffmpeg/CMakeLists.txt @@ -18,14 +18,17 @@ if (NOT FFMPEG_FOUND_SEVERAL) endif() set(ffmpeg_headers_public + ffmpeg_audio_stream_settings.h ffmpeg_cuda.h ffmpeg_init.h ffmpeg_util.h ffmpeg_video_input.h + ffmpeg_video_input_clip.h ffmpeg_video_output.h ffmpeg_video_raw_image.cxx ffmpeg_video_raw_metadata.cxx ffmpeg_video_settings.h + ffmpeg_video_uninterpreted_data.h ) kwiver_install_headers( @@ -39,14 +42,17 @@ kwiver_install_headers( ) set(ffmpeg_sources + ffmpeg_audio_stream_settings.cxx ffmpeg_cuda.cxx ffmpeg_init.cxx ffmpeg_util.cxx ffmpeg_video_input.cxx + ffmpeg_video_input_clip.cxx ffmpeg_video_output.cxx ffmpeg_video_raw_image.cxx ffmpeg_video_raw_metadata.cxx ffmpeg_video_settings.cxx + ffmpeg_video_uninterpreted_data.cxx ) if(KWIVER_ENABLE_CUDA) diff --git a/arrows/ffmpeg/ffmpeg_audio_stream_settings.cxx b/arrows/ffmpeg/ffmpeg_audio_stream_settings.cxx new file mode 100644 index 0000000000..399f6761e7 --- /dev/null +++ b/arrows/ffmpeg/ffmpeg_audio_stream_settings.cxx @@ -0,0 +1,88 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Definition of FFmpeg audio stream settings. + +#include + +namespace kwiver { + +namespace arrows { + +namespace ffmpeg { + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream_settings +::ffmpeg_audio_stream_settings() + : index{ -1 }, + parameters{ avcodec_parameters_alloc() }, + time_base{ 0, 1 } +{ + if( !parameters ) + { + throw std::runtime_error{ "Could not allocate AVCodecParameters" }; + } + parameters->codec_type = AVMEDIA_TYPE_AUDIO; +} + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream_settings +::ffmpeg_audio_stream_settings( ffmpeg_audio_stream_settings const& other ) + : parameters{} +{ + *this = other; +} + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream_settings +::ffmpeg_audio_stream_settings( ffmpeg_audio_stream_settings&& other ) + : parameters{} +{ + *this = std::move( other ); +} + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream_settings +::~ffmpeg_audio_stream_settings() +{} + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream_settings& +ffmpeg_audio_stream_settings +::operator=( ffmpeg_audio_stream_settings const& other ) +{ + index = other.index; + + parameters.reset( + throw_error_null( + avcodec_parameters_alloc(), + "Could not allocate AVCodecParameters" ) ); + + throw_error_code( + avcodec_parameters_copy( parameters.get(), other.parameters.get() ), + "Could not copy codec parameters" ); + + time_base = other.time_base; + + return *this; +} + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream_settings& +ffmpeg_audio_stream_settings +::operator=( ffmpeg_audio_stream_settings&& other ) +{ + index = std::move( other.index ); + parameters = std::move( other.parameters ); + time_base = std::move( other.time_base ); + + return *this; +} + +} // namespace ffmpeg + +} // namespace arrows + +} // namespace kwiver diff --git a/arrows/ffmpeg/ffmpeg_audio_stream_settings.h b/arrows/ffmpeg/ffmpeg_audio_stream_settings.h new file mode 100644 index 0000000000..ff372aef45 --- /dev/null +++ b/arrows/ffmpeg/ffmpeg_audio_stream_settings.h @@ -0,0 +1,61 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Declaration of FFmpeg audio stream settings. + +#ifndef KWIVER_ARROWS_FFMPEG_FFMPEG_AUDIO_STREAM_SETTINGS_H_ +#define KWIVER_ARROWS_FFMPEG_FFMPEG_AUDIO_STREAM_SETTINGS_H_ + +#include +#include + +extern "C" { +#include +} + +namespace kwiver { + +namespace arrows { + +namespace ffmpeg { + +// ---------------------------------------------------------------------------- +/// Parameters describing the general characteristics of an audio stream. +/// +/// This struct will be filled in by ffmpeg_video_input, to be used by +/// ffmpeg_video_output when creating an audio stream. Members have been left +/// public so users may modify them before passing to ffmpeg_video_output. +struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_audio_stream_settings +{ + ffmpeg_audio_stream_settings(); + ffmpeg_audio_stream_settings( ffmpeg_audio_stream_settings const& other ); + ffmpeg_audio_stream_settings( ffmpeg_audio_stream_settings&& other ); + + ~ffmpeg_audio_stream_settings(); + + ffmpeg_audio_stream_settings& + operator=( ffmpeg_audio_stream_settings const& other ); + ffmpeg_audio_stream_settings& + operator=( ffmpeg_audio_stream_settings&& other ); + + /// Index of this stream in the input video. Does not determine the index in + /// the output video. + int index; + + /// FFmpeg's parameters determining how the audio codec is set up. + codec_parameters_uptr parameters; + + /// Time base of this stream in the input video. Not guaranteed to determine + /// the time base in the output video. + AVRational time_base; +}; + +} // namespace ffmpeg + +} // namespace arrows + +} // namespace kwiver + +#endif diff --git a/arrows/ffmpeg/ffmpeg_util.cxx b/arrows/ffmpeg/ffmpeg_util.cxx index 53ebc5c74d..f680439a12 100644 --- a/arrows/ffmpeg/ffmpeg_util.cxx +++ b/arrows/ffmpeg/ffmpeg_util.cxx @@ -127,7 +127,7 @@ DEFINE_DELETER( format_context, AVFormatContext ) // ---------------------------------------------------------------------------- DEFINE_DELETER( codec_context, AVCodecContext ) { - if( ptr->codec ) + if( ptr->codec && ptr->codec_type == AVMEDIA_TYPE_VIDEO ) { avcodec_flush_buffers( ptr ); } @@ -176,6 +176,12 @@ DEFINE_DELETER( hardware_device_context, AVBufferRef ) av_buffer_unref( &ptr ); } +// ---------------------------------------------------------------------------- +DEFINE_DELETER( bsf_context, AVBSFContext ) +{ + av_bsf_free( &ptr ); +} + #undef DEFINE_DELETER } // namespace ffmpeg diff --git a/arrows/ffmpeg/ffmpeg_util.h b/arrows/ffmpeg/ffmpeg_util.h index a4bc211a99..07b45f1f56 100644 --- a/arrows/ffmpeg/ffmpeg_util.h +++ b/arrows/ffmpeg/ffmpeg_util.h @@ -14,6 +14,9 @@ extern "C" { #include +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(58, 87, 100) +#include +#endif #include #include #include @@ -92,6 +95,7 @@ DECLARE_PTRS( filter_graph, AVFilterGraph ) DECLARE_PTRS( filter_in_out, AVFilterInOut ) DECLARE_PTRS( sws_context, SwsContext ) DECLARE_PTRS( hardware_device_context, AVBufferRef ) +DECLARE_PTRS( bsf_context, AVBSFContext ) #undef DECLARE_PTRS diff --git a/arrows/ffmpeg/ffmpeg_video_input.cxx b/arrows/ffmpeg/ffmpeg_video_input.cxx index 1b27c96901..acd4a08d30 100644 --- a/arrows/ffmpeg/ffmpeg_video_input.cxx +++ b/arrows/ffmpeg/ffmpeg_video_input.cxx @@ -12,6 +12,7 @@ #include "ffmpeg_video_raw_image.h" #include "ffmpeg_video_raw_metadata.h" #include "ffmpeg_video_settings.h" +#include "ffmpeg_video_uninterpreted_data.h" #include #include @@ -73,6 +74,8 @@ struct ffmpeg_klv_stream ffmpeg_klv_stream( ffmpeg_klv_stream const& ) = delete; ffmpeg_klv_stream( ffmpeg_klv_stream&& ) = delete; + klv::klv_stream_settings settings() const; + void send_packet( AVPacket* packet ); void advance( std::optional< uint64_t > backup_timestamp, @@ -84,6 +87,7 @@ struct ffmpeg_klv_stream AVStream* stream; std::vector< packet_uptr > buffer; + std::vector< packet_uptr > this_frame_buffer; std::vector< uint8_t > bytes; std::vector< klv::klv_packet > packets; klv::klv_timeline timeline; @@ -112,6 +116,34 @@ ::ffmpeg_klv_stream( AVStream* stream ) } } +// ---------------------------------------------------------------------------- +klv::klv_stream_settings +ffmpeg_klv_stream +::settings() const +{ + klv::klv_stream_settings result; + result.index = stream->index; + switch( stream->codecpar->profile ) + { +#if LIBAVCODEC_VERSION_MAJOR > 57 + case FF_PROFILE_KLVA_SYNC: + result.type = klv::KLV_STREAM_TYPE_SYNC; + break; + case FF_PROFILE_KLVA_ASYNC: + result.type = klv::KLV_STREAM_TYPE_ASYNC; + break; +#endif + default: + LOG_DEBUG( + kv::get_logger( "klv" ), + "Could not determine synchronicity of KLV stream " << stream->index + << "; reporting as asynchronous" ); + result.type = klv::KLV_STREAM_TYPE_ASYNC; + break; + } + return result; +} + // ---------------------------------------------------------------------------- void ffmpeg_klv_stream @@ -165,6 +197,7 @@ ffmpeg_klv_stream ::advance( std::optional< uint64_t > backup_timestamp, int64_t max_pts, int64_t max_pos ) { + this_frame_buffer.clear(); packets.clear(); for( auto it = buffer.begin(); it != buffer.end(); ) @@ -174,6 +207,7 @@ ::advance( ( packet.pts == AV_NOPTS_VALUE && packet.pos <= max_pos ) ) { bytes.insert( bytes.end(), packet.data, packet.data + packet.size ); + this_frame_buffer.emplace_back( std::move( *it ) ); it = buffer.erase( it ); } else @@ -257,15 +291,58 @@ ::vital_metadata( uint64_t timestamp, bool smooth_packets ) klv_result.add< kv::VITAL_META_METADATA_ORIGIN >( "KLV" ); klv_result.add< kv::VITAL_META_VIDEO_DATA_STREAM_INDEX >( stream->index ); -#if LIBAVCODEC_VERSION_MAJOR > 57 - if( stream->codecpar->profile >= 0 ) + klv_result.add< kv::VITAL_META_VIDEO_DATA_STREAM_SYNCHRONOUS >( + settings().type == klv::KLV_STREAM_TYPE_SYNC + ); + + return result; +} + +// ---------------------------------------------------------------------------- +struct ffmpeg_audio_stream +{ + ffmpeg_audio_stream( AVStream* stream ); + + ffmpeg_audio_stream( ffmpeg_audio_stream const& ) = delete; + ffmpeg_audio_stream( ffmpeg_audio_stream&& ) = delete; + + ffmpeg_audio_stream_settings settings() const; + + AVStream* stream; + codec_context_uptr codec_context; +}; + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream +::ffmpeg_audio_stream( AVStream* stream ) : stream{ stream } +{ + if( !stream ) { - klv_result.add< kv::VITAL_META_VIDEO_DATA_STREAM_SYNCHRONOUS >( - stream->codecpar->profile == FF_PROFILE_KLVA_SYNC - ); + throw std::logic_error( "ffmpeg_audio_stream given null stream" ); } -#endif + auto const codec = + throw_error_null( + avcodec_find_decoder( stream->codecpar->codec_id ), + "Could not find audio decoder" ); + + codec_context.reset( + throw_error_null( avcodec_alloc_context3( codec ), + "Could not allocate codec context" ) ); +} + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream_settings +ffmpeg_audio_stream +::settings() const +{ + ffmpeg_audio_stream_settings result; + result.index = stream->index; + throw_error_code( + avcodec_parameters_copy( + result.parameters.get(), stream->codecpar ), + "Could not copy codec parameters" ); + result.time_base = stream->time_base; return result; } @@ -291,6 +368,10 @@ class ffmpeg_video_input::priv kv::image_container_sptr convert_image(); kv::metadata_vector const& convert_metadata(); + ffmpeg_video_raw_image& get_raw_image(); + ffmpeg_video_raw_metadata& get_raw_metadata(); + ffmpeg_video_uninterpreted_data& get_uninterpreted_data(); + open_video_state* parent; kv::logger_handle_t logger; @@ -299,22 +380,39 @@ class ffmpeg_video_input::priv kv::image_memory_sptr image_memory; kv::image_container_sptr image; - ffmpeg_video_raw_image_sptr raw_image; + kv::video_raw_image_sptr raw_image; std::optional< kv::metadata_vector > metadata; - ffmpeg_video_raw_metadata_sptr raw_metadata; + kv::video_raw_metadata_sptr raw_metadata; + + kv::video_uninterpreted_data_sptr uninterpreted_data; bool is_draining; }; struct open_video_state { + struct filter_parameters { + explicit filter_parameters( AVCodecContext const& codec_context ); + explicit filter_parameters( AVFrame const& frame ); + + bool operator==( filter_parameters const& other ) const; + bool operator!=( filter_parameters const& other ) const; + + int width; + int height; + AVPixelFormat pix_fmt; + AVRational sample_aspect_ratio; + }; + open_video_state( priv& parent, std::string const& path ); ~open_video_state(); bool try_codec(); - void init_filters(); - bool advance(); - void seek( kv::frame_id_t frame_number ); + void init_filters( filter_parameters const& parameters ); + bool advance( bool is_first_frame_of_seek = false ); + void clear_state_for_seek(); + void seek_to_start(); + void seek( kv::frame_id_t frame_number, seek_mode mode ); void set_video_metadata( kv::metadata& md ); double curr_time() const; double duration() const; @@ -338,17 +436,28 @@ class ffmpeg_video_input::priv filter_graph_uptr filter_graph; AVFilterContext* filter_sink_context; AVFilterContext* filter_source_context; + std::optional< filter_parameters > filter_params; sws_context_uptr image_conversion_context; + std::optional< kv::frame_id_t > frame_count; int64_t start_ts; + AVRational maybe_frame_rate; std::map< int64_t, klv::misp_timestamp > pts_to_misp_ts; + std::map< int64_t, int64_t > packet_pos_to_dts; + int64_t prev_frame_dts; + int64_t prev_video_dts; + std::multimap< int64_t, packet_uptr > lookahead; + std::list< packet_uptr > raw_image_buffer; std::list< ffmpeg_klv_stream > klv_streams; kv::metadata_map_sptr all_metadata; + std::list< ffmpeg_audio_stream > audio_streams; + std::optional< frame_state > frame; + bool lookahead_at_eof; bool at_eof; }; @@ -357,8 +466,8 @@ class ffmpeg_video_input::priv hardware_device_context_uptr hardware_device_context; - bool imagery_enabled; bool klv_enabled; + bool audio_enabled; bool use_misp_timestamps; bool smooth_klv_packets; std::string unknown_stream_behavior; @@ -393,8 +502,8 @@ ::priv( ffmpeg_video_input& parent ) : parent( parent ), logger{ kv::get_logger( "ffmpeg_video_input" ) }, hardware_device_context{ nullptr }, - imagery_enabled{ true }, klv_enabled{ true }, + audio_enabled{ true }, use_misp_timestamps{ false }, smooth_klv_packets{ false }, unknown_stream_behavior{ "klv" }, @@ -533,6 +642,7 @@ ::frame_state( open_video_state& parent ) raw_image{}, metadata{}, raw_metadata{}, + uninterpreted_data{}, is_draining{ false } { // Allocate frame containers @@ -544,6 +654,7 @@ ::frame_state( open_video_state& parent ) // Allocate raw data containers raw_image.reset( new ffmpeg_video_raw_image{} ); raw_metadata.reset( new ffmpeg_video_raw_metadata{} ); + uninterpreted_data.reset( new ffmpeg_video_uninterpreted_data{} ); } // ---------------------------------------------------------------------------- @@ -556,11 +667,6 @@ kv::image_container_sptr ffmpeg_video_input::priv::frame_state ::convert_image() { - if( !parent->parent->imagery_enabled ) - { - return nullptr; - } - if( image ) { return image; @@ -582,6 +688,13 @@ ::convert_image() // Run the frame through the filter graph if( parent->filter_source_context && parent->filter_sink_context ) { + // Check for parameter changes + open_video_state::filter_parameters const frame_params{ *frame }; + if( frame_params != *parent->filter_params ) + { + parent->init_filters( frame_params ); + } + int recv_err; do{ throw_error_code( @@ -722,6 +835,73 @@ ::convert_metadata() return *metadata; } +// ---------------------------------------------------------------------------- +ffmpeg_video_raw_image& +ffmpeg_video_input::priv::frame_state +::get_raw_image() +{ + return dynamic_cast< ffmpeg_video_raw_image& >( *raw_image ); +} + +// ---------------------------------------------------------------------------- +ffmpeg_video_raw_metadata& +ffmpeg_video_input::priv::frame_state +::get_raw_metadata() +{ + return dynamic_cast< ffmpeg_video_raw_metadata& >( *raw_metadata ); +} + +// ---------------------------------------------------------------------------- +ffmpeg_video_uninterpreted_data& +ffmpeg_video_input::priv::frame_state +::get_uninterpreted_data() +{ + return + dynamic_cast< ffmpeg_video_uninterpreted_data& >( *uninterpreted_data ); +} + +// ---------------------------------------------------------------------------- +ffmpeg_video_input::priv::open_video_state::filter_parameters +::filter_parameters( AVCodecContext const& codec_context ) + : width{ codec_context.width }, + height{ codec_context.height }, + pix_fmt{ + codec_context.hw_device_ctx + ? codec_context.sw_pix_fmt + : codec_context.pix_fmt }, + sample_aspect_ratio{ codec_context.sample_aspect_ratio } +{} + +// ---------------------------------------------------------------------------- +ffmpeg_video_input::priv::open_video_state::filter_parameters +::filter_parameters( AVFrame const& frame ) + : width{ frame.width }, + height{ frame.height }, + pix_fmt{ static_cast< AVPixelFormat >( frame.format ) }, + sample_aspect_ratio{ frame.sample_aspect_ratio } +{} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input::priv::open_video_state::filter_parameters +::operator==( filter_parameters const& other ) const +{ + return + width == other.width && + height == other.height && + pix_fmt == other.pix_fmt && + sample_aspect_ratio.num == other.sample_aspect_ratio.num && + sample_aspect_ratio.den == other.sample_aspect_ratio.den; +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input::priv::open_video_state::filter_parameters +::operator!=( filter_parameters const& other ) const +{ + return !( *this == other ); +} + // ---------------------------------------------------------------------------- ffmpeg_video_input::priv::open_video_state ::open_video_state( priv& parent, std::string const& path ) @@ -736,45 +916,46 @@ ::open_video_state( priv& parent, std::string const& path ) filter_sink_context{ nullptr }, filter_source_context{ nullptr }, image_conversion_context{ nullptr }, + frame_count{}, start_ts{ 0 }, + maybe_frame_rate{ 0, 0 }, pts_to_misp_ts{}, + packet_pos_to_dts{}, + prev_frame_dts{ AV_NOPTS_VALUE }, + prev_video_dts{ AV_NOPTS_VALUE }, + lookahead{}, + raw_image_buffer{}, klv_streams{}, all_metadata{ nullptr }, + audio_streams{}, frame{}, + lookahead_at_eof{ false }, at_eof{ false } { - // Open the file - { - AVFormatContext* ptr = nullptr; - throw_error_code( - avformat_open_input( &ptr, path.c_str(), NULL, NULL ), - "Could not open input stream" ); - format_context.reset( ptr ); - } - - // Try to probe the file for stream information - constexpr size_t max_probe_tries = 3; - format_context->probesize = 5'000'000; // 5 MB - format_context->max_analyze_duration = 10 * AV_TIME_BASE; // 10 seconds +// Try to probe the file for stream information + constexpr size_t max_probe_tries = 4; + int64_t probesize = 5'000'000; // 5 MB + int64_t max_analyze_duration = 10 * AV_TIME_BASE; // 10 seconds + uint64_t increase_factor = 100; for( auto const i : kvr::iota( max_probe_tries ) ) { - // Increase how much of file to analyze on later attempts - if( i != 0 ) + // Open the file { - LOG_ERROR( - logger, - "Could not find a valid video stream in the input on attempt " << i - << " of " << max_probe_tries ); - format_context->probesize *= 3; - format_context->max_analyze_duration *= 3; + AVFormatContext* ptr = nullptr; + throw_error_code( + avformat_open_input( &ptr, path.c_str(), NULL, NULL ), + "Could not open input stream" ); + format_context.reset( ptr ); } + format_context->probesize = probesize; + format_context->max_analyze_duration = max_analyze_duration; // Get the stream information by reading a bit of the file throw_error_code( avformat_find_stream_info( format_context.get(), NULL ), "Could not read stream information" ); - // Find a video stream, and optionally any data streams + // Find a video stream, and optionally any data or audio streams for( auto const j : kvr::iota( format_context->nb_streams ) ) { auto const stream = format_context->streams[ j ]; @@ -820,6 +1001,12 @@ ::open_video_state( priv& parent, std::string const& path ) LOG_INFO( logger, "Ignoring unknown stream " << stream->index ); } } + else if( + parent.audio_enabled && + params->codec_type == AVMEDIA_TYPE_AUDIO ) + { + audio_streams.emplace_back( stream ); + } } if( video_stream ) @@ -827,6 +1014,17 @@ ::open_video_state( priv& parent, std::string const& path ) // Success! break; } + + // Increase how much of file to analyze on later attempts + LOG_ERROR( + logger, + "Could not find a valid video stream in the input on attempt " + << ( i + 1 ) << " of " << max_probe_tries ); + probesize *= increase_factor; + max_analyze_duration *= increase_factor; + + // Clear state + klv_streams.clear(); } // Confirm stream characteristics @@ -844,74 +1042,71 @@ ::open_video_state( priv& parent, std::string const& path ) } } - if( parent.imagery_enabled ) - { - // Dig up information about the video's codec - auto const video_params = video_stream->codecpar; - auto const codec_id = video_params->codec_id; - LOG_INFO( - logger, "Video requires codec type: " << pretty_codec_name( codec_id ) ); - - // Codec prioritization scheme: - // (1) Choose hardware over software codecs - auto const codec_cmp = - [ & ]( AVCodec const* lhs, AVCodec const* rhs ) -> bool { - return - std::make_tuple( is_hardware_codec( lhs ) ) > - std::make_tuple( is_hardware_codec( rhs ) ); - }; - std::multiset< - AVCodec const*, std::function< bool( AVCodec const*, AVCodec const* ) > > - possible_codecs{ codec_cmp }; - - // Find all compatible CUDA codecs + // Dig up information about the video's codec + auto const video_params = video_stream->codecpar; + auto const codec_id = video_params->codec_id; + LOG_INFO( + logger, "Video requires codec type: " << pretty_codec_name( codec_id ) ); + + // Codec prioritization scheme: + // (1) Choose hardware over software codecs + auto const codec_cmp = + [ & ]( AVCodec const* lhs, AVCodec const* rhs ) -> bool { + return + std::make_tuple( is_hardware_codec( lhs ) ) > + std::make_tuple( is_hardware_codec( rhs ) ); + }; + std::multiset< + AVCodec const*, std::function< bool( AVCodec const*, AVCodec const* ) > > + possible_codecs{ codec_cmp }; + + // Find all compatible CUDA codecs #ifdef KWIVER_ENABLE_FFMPEG_CUDA - if( parent.cuda_device() ) - { - auto const cuda_codecs = cuda_find_decoders( *video_params ); - possible_codecs.insert( cuda_codecs.begin(), cuda_codecs.end() ); - } + if( parent.cuda_device() ) + { + auto const cuda_codecs = cuda_find_decoders( *video_params ); + possible_codecs.insert( cuda_codecs.begin(), cuda_codecs.end() ); + } #endif - // Find all compatible software codecs - AVCodec const* codec_ptr = nullptr; + // Find all compatible software codecs + AVCodec const* codec_ptr = nullptr; #if LIBAVCODEC_VERSION_MAJOR > 57 - for( void* it = nullptr; ( codec_ptr = av_codec_iterate( &it ) ); ) + for( void* it = nullptr; ( codec_ptr = av_codec_iterate( &it ) ); ) #else - while( ( codec_ptr = av_codec_next( codec_ptr ) ) ) + while( ( codec_ptr = av_codec_next( codec_ptr ) ) ) #endif + { + if( codec_ptr->id == codec_id && + av_codec_is_decoder( codec_ptr ) && + !is_hardware_codec( codec_ptr ) && + !( codec_ptr->capabilities & AV_CODEC_CAP_EXPERIMENTAL ) ) { - if( codec_ptr->id == codec_id && - av_codec_is_decoder( codec_ptr ) && - !is_hardware_codec( codec_ptr ) && - !( codec_ptr->capabilities & AV_CODEC_CAP_EXPERIMENTAL ) ) - { - possible_codecs.emplace( codec_ptr ); - } + possible_codecs.emplace( codec_ptr ); } + } - // Find the first compatible codec that works, in priority order - for( auto const possible_codec : possible_codecs ) + // Find the first compatible codec that works, in priority order + for( auto const possible_codec : possible_codecs ) + { + codec = possible_codec; + if( try_codec() ) { - codec = possible_codec; - if( try_codec() ) - { - break; - } - else - { - codec = nullptr; - } + break; + } + else + { + codec = nullptr; } - - throw_error_null( - codec, - "Could not open video with any known input codec. ", - possible_codecs.size(), " codecs were tried. ", - "Required codec type: ", pretty_codec_name( codec_id ) ); - LOG_INFO( - logger, "Successfully loaded codec: " << pretty_codec_name( codec ) ); } + + throw_error_null( + codec, + "Could not open video with any known input codec. ", + possible_codecs.size(), " codecs were tried. ", + "Required codec type: ", pretty_codec_name( codec_id ) ); + LOG_INFO( + logger, "Successfully loaded codec: " << pretty_codec_name( codec ) ); } // ---------------------------------------------------------------------------- @@ -947,6 +1142,9 @@ ::try_codec() av_buffer_ref( parent->hardware_device_context.get() ); } + codec_context->thread_count = 0; + codec_context->thread_type = FF_THREAD_FRAME; + // Open codec auto const err = avcodec_open2( codec_context.get(), codec, NULL ); if( err < 0 ) @@ -959,13 +1157,7 @@ ::try_codec() } // Initialize filter graph - init_filters(); - - // Start time taken from the first decodable frame - throw_error_code( - av_seek_frame( - format_context.get(), video_stream->index, 0, AVSEEK_FLAG_FRAME ), - "Could not seek to beginning of video" ); + init_filters( filter_parameters{ *codec_context } ); // Read frames until we can successfully decode one to get start timestamp { @@ -1000,24 +1192,14 @@ ::try_codec() } av_packet_unref( tmp_packet.get() ); } while( send_err || recv_err ); + auto const duration_q = + AVRational{ static_cast< int >( tmp_frame->pkt_duration ), 1 }; + maybe_frame_rate = + av_inv_q( av_mul_q( duration_q, video_stream->time_base ) ); start_ts = tmp_frame->best_effort_timestamp; } - // Seek back to start - auto seek_err = - av_seek_frame( - format_context.get(), -1, INT64_MIN, - AVSEEK_FLAG_BYTE | AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY ); - if( seek_err < 0 ) - { - // Sometimes seeking by byte position is not allowed, so try by timestamp - throw_error_code( - av_seek_frame( - format_context.get(), -1, INT64_MIN, - AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY ), - "Could not seek to beginning of video" ); - } - avcodec_flush_buffers( codec_context.get() ); + seek_to_start(); return true; } @@ -1025,7 +1207,7 @@ ::try_codec() // ---------------------------------------------------------------------------- void ffmpeg_video_input::priv::open_video_state -::init_filters() +::init_filters( filter_parameters const& parameters ) { // Check for empty filter string if( std::all_of( @@ -1042,18 +1224,14 @@ ::init_filters() // Create the input buffer { - auto const pix_fmt = - codec_context->hw_device_ctx - ? codec_context->sw_pix_fmt - : codec_context->pix_fmt; std::stringstream ss; - ss << "video_size=" << codec_context->width << "x" - << codec_context->height - << ":pix_fmt=" << pix_fmt + ss << "video_size=" << parameters.width << "x" + << parameters.height + << ":pix_fmt=" << parameters.pix_fmt << ":time_base=" << video_stream->time_base.num << "/" << video_stream->time_base.den - << ":pixel_aspect=" << codec_context->sample_aspect_ratio.num << "/" - << codec_context->sample_aspect_ratio.den; + << ":pixel_aspect=" << parameters.sample_aspect_ratio.num << "/" + << parameters.sample_aspect_ratio.den; throw_error_code( avfilter_graph_create_filter( &filter_source_context, avfilter_get_by_name( "buffer" ), @@ -1105,12 +1283,14 @@ ::init_filters() throw_error_code( avfilter_graph_config( filter_graph.get(), NULL ), "Could not configure filter graph" ); + + filter_params.emplace( parameters ); } // ---------------------------------------------------------------------------- bool ffmpeg_video_input::priv::open_video_state -::advance() +::advance( bool is_first_frame_of_seek ) { if( at_eof ) { @@ -1127,118 +1307,404 @@ ::advance() frame.reset(); // Run through video until we can assemble a frame image - packet_uptr packet{ - throw_error_null( av_packet_alloc(), "Could not allocate packet" ) }; - while( !at_eof && !frame.has_value() ) + std::vector< int64_t > video_pos_list; + while( !frame.has_value() && !at_eof ) { - if( !new_frame.is_draining ) + // We need at least one video packet before we could expect another frame + auto first_video_it = lookahead.end(); + + // We want to make sure each stream has caught up before actually using the + // next video packet + std::vector< int64_t > most_recent_dts( + format_context->nb_streams, AV_NOPTS_VALUE ); + + // Take stock of packets we have in storage + for( auto it = lookahead.begin(); it != lookahead.end(); ++it ) + { + most_recent_dts.at( it->second->stream_index ) = + std::max( + most_recent_dts.at( it->second->stream_index ), it->first ); + if( first_video_it == lookahead.end() && + it->second->stream_index == video_stream->index ) + { + first_video_it = it; + } + } + + // Functor determining if we need to parse more of other streams before + // continuing with decoding the video + auto const looked_ahead_enough = + [ this, &most_recent_dts, &first_video_it ]() -> bool + { + if( first_video_it == lookahead.end() ) + { + return false; + } + + auto const first_video_pts = + av_rescale_q( + first_video_it->second->pts, + video_stream->time_base, + AVRational{ 1, AV_TIME_BASE } ); + + auto const first_video_end = + ( first_video_it->second->duration <= 0 ) + ? first_video_pts + : av_rescale_q( + first_video_it->second->pts + first_video_it->second->duration, + video_stream->time_base, + AVRational{ 1, AV_TIME_BASE } ); + + for( auto const& stream : klv_streams ) + { + auto const dts = most_recent_dts.at( stream.stream->index ); + if( dts == AV_NOPTS_VALUE || + dts <= first_video_it->first || + dts <= first_video_pts ) + { + return false; + } + } + + for( auto const& stream : audio_streams ) + { + auto const dts = most_recent_dts.at( stream.stream->index ); + if( dts == AV_NOPTS_VALUE || + dts <= first_video_it->first || + dts < first_video_end ) + { + return false; + } + } + + return true; + }; + + // Read until all streams are up-to-date with the video stream + packet_uptr packet{ + throw_error_null( av_packet_alloc(), "Could not allocate packet" ) }; + while( !lookahead_at_eof && !looked_ahead_enough() ) { - // Read next packet av_packet_unref( packet.get() ); + auto const read_err = av_read_frame( format_context.get(), packet.get() ); if( read_err == AVERROR_EOF ) { - // End of input. Tell this to decoder - if( parent->imagery_enabled ) + // End of input + lookahead_at_eof = true; + break; + } + throw_error_code( read_err, "Could not read next packet from file" ); + + // Sanity check for stream index + if( packet->stream_index < 0 || + static_cast< unsigned int >( packet->stream_index ) >= + most_recent_dts.size() ) + { + continue; + } + + // Process video packet + if( packet->stream_index == video_stream->index ) + { + // Need pts + if( packet->pts == AV_NOPTS_VALUE ) { - avcodec_send_packet( codec_context.get(), nullptr ); - new_frame.is_draining = true; + LOG_ERROR( + parent->logger, + "Dropping video packet with invalid pts" ); + continue; } - else + + // Replace any weird dts with a guess + if( packet->dts == AV_NOPTS_VALUE ) { - at_eof = true; - return false; + packet->dts = + ( prev_video_dts == AV_NOPTS_VALUE ) + ? 0 + : ( prev_video_dts + 1 ); } + else if( + packet->dts == AV_NOPTS_VALUE || + packet->dts < prev_video_dts || + packet->dts > packet->pts ) + { + if( prev_video_dts != AV_NOPTS_VALUE && packet->duration > 0 ) + { + packet->dts = prev_video_dts + packet->duration; + packet->dts = std::min( packet->dts, packet->pts ); + } + else + { + packet->dts = packet->pts; + } + } + prev_video_dts = packet->dts; } - else + + // Guess the missing DTS field for asynchronous KLV + auto packet_dts = + av_rescale_q( + packet->dts, + format_context->streams[ packet->stream_index ]->time_base, + AVRational{ 1, AV_TIME_BASE } ); + for( auto& stream : klv_streams ) { - throw_error_code( read_err, "Could not read next packet from file" ); + if( packet->stream_index != stream.stream->index || + stream.settings().type != klv::KLV_STREAM_TYPE_ASYNC ) + { + continue; + } + + packet_dts = + lookahead.empty() ? 0 : std::prev( lookahead.end() )->first; + break; + } + + // Put the packet in the lookahead buffer + auto new_packet = + throw_error_null( av_packet_alloc(), "Could not allocate packet" ); + auto const it = lookahead.emplace( packet_dts, std::move( new_packet ) ); + av_packet_move_ref( it->second.get(), packet.get() ); + + // Update stats for this stream + most_recent_dts.at( it->second->stream_index ) = + std::max( + packet_dts, most_recent_dts.at( it->second->stream_index ) ); + if( first_video_it == lookahead.end() && + it->second->stream_index == video_stream->index ) + { + first_video_it = it; + } + } + + // Couldn't find next video packet? Tell the decoder to flush any remaining + // buffered frames + if( first_video_it == lookahead.end() && + lookahead_at_eof && !new_frame.is_draining ) + { + avcodec_send_packet( codec_context.get(), nullptr ); + new_frame.is_draining = true; + } + + // Process next video packet, if there is one + if( first_video_it != lookahead.end() ) + { + packet = std::move( first_video_it->second ); + lookahead.erase( first_video_it ); + first_video_it = lookahead.end(); + video_pos_list.emplace_back( packet->pos ); + + // Record packet as raw image + raw_image_buffer.emplace_back( + throw_error_null( + av_packet_alloc(), "Could not allocate packet" ) ); + throw_error_code( + av_packet_ref( raw_image_buffer.back().get(), packet.get() ), + "Could not give packet to raw image cache" ); + packet_pos_to_dts.emplace( packet->pos, packet->dts ); + + // Find MISP timestamp + for( auto const tag_type : { klv::MISP_TIMESTAMP_TAG_STRING, + klv::MISP_TIMESTAMP_TAG_UUID } ) + { + auto it = + klv::find_misp_timestamp( + packet->data, packet->data + packet->size, tag_type ); + if( it != packet->data + packet->size ) + { + auto const timestamp = klv::read_misp_timestamp( it ); + pts_to_misp_ts.emplace( packet->pts, timestamp ); + break; + } + } + + // Send packet to decoder + auto const send_err = + avcodec_send_packet( codec_context.get(), packet.get() ); + if( send_err != AVERROR_INVALIDDATA ) + { + throw_error_code( send_err, "Decoder rejected packet" ); + } + } + + // Receive decoded frame + auto const recv_err = + avcodec_receive_frame( codec_context.get(), new_frame.frame.get() ); + auto dts_lookup_failed = false; + switch( recv_err ) + { + case 0: + // Success + frame = std::move( new_frame ); + if( frame_count ) + { + ++( *frame_count ); + } - // Video packet - if( parent->imagery_enabled && - packet->stream_index == video_stream->index ) + // Look up the dts of the packet that contained this frame + if( auto const it = packet_pos_to_dts.find( frame->frame->pkt_pos ); + it != packet_pos_to_dts.end() ) { - // Record packet as raw image - new_frame.raw_image->packets.emplace_back( - throw_error_null( - av_packet_alloc(), "Could not allocate packet" ) ); - throw_error_code( - av_packet_ref( - new_frame.raw_image->packets.back().get(), packet.get() ), - "Could not give packet to raw image cache" ); - - // Find MISP timestamp - for( auto const tag_type : { klv::MISP_TIMESTAMP_TAG_STRING, - klv::MISP_TIMESTAMP_TAG_UUID } ) + for( auto jt = raw_image_buffer.begin(); + jt != raw_image_buffer.end(); ) { - auto it = - klv::find_misp_timestamp( - packet->data, packet->data + packet->size, tag_type ); - if( it != packet->data + packet->size ) + if( ( *jt )->dts <= it->second || + ( *jt )->dts <= frame->frame->pkt_dts ) + { + auto const next_jt = std::next( jt ); + frame->get_raw_image().packets.splice( + frame->get_raw_image().packets.end(), raw_image_buffer, jt ); + jt = next_jt; + } + else { - auto const timestamp = klv::read_misp_timestamp( it ); - pts_to_misp_ts.emplace( packet->pts, timestamp ); - break; + ++jt; } } + frame->get_raw_image().frame_dts = it->second; + prev_frame_dts = it->second; + packet_pos_to_dts.erase( it ); + } + else + { + LOG_DEBUG( + parent->logger, + "Raw frame dts lookup failed, likely due to corruption" ); + dts_lookup_failed = true; + frame->get_raw_image().frame_dts = prev_frame_dts; + + // We can't erase an entry from packet_pos_to_dts, since we don't + // know for sure which entry to erase + } + frame->get_raw_image().frame_pts = frame->frame->best_effort_timestamp; + frame->get_raw_image().is_keyframe = frame->frame->key_frame; - // Send packet to decoder - auto const send_err = - avcodec_send_packet( codec_context.get(), packet.get() ); - if( send_err != AVERROR_INVALIDDATA ) + // Clean up + if( frame->frame->key_frame && !dts_lookup_failed ) + { + auto const it = + packet_pos_to_dts.lower_bound( frame->frame->pkt_pos ); + if( it != packet_pos_to_dts.begin() ) { - throw_error_code( send_err, "Decoder rejected packet" ); + auto const cleanup_count = + std::distance( packet_pos_to_dts.begin(), it ); + LOG_DEBUG( + parent->logger, + "Cleaning up " << cleanup_count << " dts lookup entries" ); + packet_pos_to_dts.erase( packet_pos_to_dts.begin(), it ); } } + break; + case AVERROR_EOF: + // End of file + at_eof = true; + break; + case AVERROR_INVALIDDATA: + case AVERROR( EAGAIN ): + // Acceptable errors + break; + default: + // Unacceptable errors + throw_error_code( recv_err, "Decoder returned error" ); + break; + } + } - // KLV packet - for( auto& stream : klv_streams ) + if( frame.has_value() ) + { + // Give the non-video streams all packets up to this new frame image + for( auto it = lookahead.begin(); it != lookahead.end(); ) + { + auto const packet_pts = + av_rescale_q( + it->second->pts, + format_context->streams[ it->second->stream_index ]->time_base, + AVRational{ 1, AV_TIME_BASE } ); + auto const frame_pts = + av_rescale_q( + frame->frame->best_effort_timestamp, + video_stream->time_base, + AVRational{ 1, AV_TIME_BASE } ); + auto const frame_minus_one_pts = + av_rescale_q( + frame->frame->best_effort_timestamp - frame->frame->pkt_duration, + video_stream->time_base, + AVRational{ 1, AV_TIME_BASE } ); + auto const frame_plus_one_pts = + av_rescale_q( + frame->frame->best_effort_timestamp + frame->frame->pkt_duration, + video_stream->time_base, + AVRational{ 1, AV_TIME_BASE } ); + + int64_t min_pos = AV_NOPTS_VALUE; + if( auto const pos_it = + std::lower_bound( + video_pos_list.begin(), video_pos_list.end(), + frame->frame->pkt_pos ); + pos_it != video_pos_list.end() && pos_it != video_pos_list.begin() ) + { + min_pos = *std::prev( pos_it ); + } + + auto found = false; + for( auto& stream : klv_streams ) + { + if( it->second->stream_index != stream.stream->index ) { - if( packet->stream_index != stream.stream->index ) - { - continue; - } + continue; + } - // Record packet as raw KLV - new_frame.raw_metadata->packets.emplace_back( - throw_error_null( - av_packet_alloc(), "Could not allocate packet" ) ); - throw_error_code( - av_packet_ref( - new_frame.raw_metadata->packets.back().get(), packet.get() ), - "Could not give packet to raw metadata cache" ); - - // Decode packet - stream.send_packet( packet.get() ); + found = true; + if( packet_pts > frame_pts ) + { + ++it; break; } + + if( !is_first_frame_of_seek || this->frame_number() == 0 || + packet_pts >= frame_minus_one_pts || + ( it->second->pts == AV_NOPTS_VALUE && + it->second->pos >= min_pos ) ) + { + stream.send_packet( it->second.get() ); + } + it = lookahead.erase( it ); + break; } - } - if( parent->imagery_enabled ) - { - // Receive decoded frame - auto const recv_err = - avcodec_receive_frame( codec_context.get(), new_frame.frame.get() ); - switch( recv_err ) + if( found ) { - case 0: - // Success - frame = std::move( new_frame ); - break; - case AVERROR_EOF: - // End of file - at_eof = true; - break; - case AVERROR_INVALIDDATA: - case AVERROR( EAGAIN ): - // Acceptable errors - break; - default: - // Unacceptable errors - throw_error_code( recv_err, "Decoder returned error" ); + continue; + } + + for( auto& stream : audio_streams ) + { + if( it->second->stream_index != stream.stream->index ) + { + continue; + } + + found = true; + if( packet_pts > frame_plus_one_pts ) + { + ++it; break; + } + + if( !is_first_frame_of_seek || this->frame_number() == 0 ) + { + frame->get_uninterpreted_data().audio_packets.emplace_back( + std::move( it->second ) ); + } + it = lookahead.erase( it ); + break; + } + + if( !found ) + { + ++it; } } } @@ -1254,33 +1720,76 @@ ::advance() auto max_pos = INT64_MAX; if( frame.has_value() ) { - max_pts = frame->frame->best_effort_timestamp; + max_pts = av_rescale_q( + frame->frame->best_effort_timestamp, + video_stream->time_base, + stream.stream->time_base ); max_pos = frame->frame->pkt_pos; } stream.advance( backup_timestamp, max_pts, max_pos ); } - return !parent->imagery_enabled || frame.has_value(); + return frame.has_value(); } // ---------------------------------------------------------------------------- void ffmpeg_video_input::priv::open_video_state -::seek( kv::frame_id_t frame_number ) +::clear_state_for_seek() { - if( frame_number == this->frame_number() ) - { - return; - } - // Clear current state + frame_count.reset(); + prev_frame_dts = AV_NOPTS_VALUE; + prev_video_dts = AV_NOPTS_VALUE; + lookahead.clear(); + raw_image_buffer.clear(); + lookahead_at_eof = false; at_eof = false; frame.reset(); for( auto& stream : klv_streams ) { stream.reset(); } +} + +// ---------------------------------------------------------------------------- +void +ffmpeg_video_input::priv::open_video_state +::seek_to_start() +{ + clear_state_for_seek(); + frame_count.emplace( -1 ); + + auto const err = + av_seek_frame( + format_context.get(), -1, INT64_MIN, + AVSEEK_FLAG_BYTE | AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY ); + if( err < 0 ) + { + // Sometimes seeking by byte position is not allowed, so try by timestamp + throw_error_code( + av_seek_frame( + format_context.get(), -1, INT64_MIN, + AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY ), + "Could not seek to beginning of video" ); + } + + if( codec_context ) + { + avcodec_flush_buffers( codec_context.get() ); + } +} + +// ---------------------------------------------------------------------------- +void +ffmpeg_video_input::priv::open_video_state +::seek( kv::frame_id_t frame_number, seek_mode mode ) +{ + if( frame_number == this->frame_number() ) + { + return; + } // Get to the desired frame by seeking some number of frames before it, then // iterating forward. If we still don't have a frame image for that frame, @@ -1289,23 +1798,20 @@ ::seek( kv::frame_id_t frame_number ) // more frames than that. auto const backstep_size = codec_context->gop_size ? codec_context->gop_size : 12; - constexpr size_t maximum_attempts = 5; - for( size_t i = 0; i < maximum_attempts; ++i ) + constexpr size_t maximum_attempts = 7; + for( size_t i = 0; i < maximum_attempts && frame_rate().num > 0; ++i ) { // Increasing backstep intervals on further tries - size_t const backstep = i ? ( ( 1 << ( i - 1 ) ) * backstep_size ) : 1; + size_t const backstep = i ? ( ( 1 << ( i - 1 ) ) * backstep_size ) : 0; // Determine timestamp from frame number - int64_t converted_timestamp = frame_number - backstep; - converted_timestamp = - av_rescale( converted_timestamp, frame_rate().den, frame_rate().num ); - converted_timestamp = - av_rescale( - converted_timestamp, - video_stream->time_base.den, video_stream->time_base.num ); - converted_timestamp += start_ts; + auto converted_timestamp = + av_rescale_q( + frame_number - backstep, av_inv_q( frame_rate() ), + video_stream->time_base ) + start_ts; // Do the seek + clear_state_for_seek(); throw_error_code( av_seek_frame( format_context.get(), video_stream->index, converted_timestamp, @@ -1317,31 +1823,156 @@ ::seek( kv::frame_id_t frame_number ) } // Move forward through frames until we get to the desired frame + size_t advance_count = 0; do { - advance(); + advance( advance_count == 0 ); + ++advance_count; if( at_eof ) { throw_error( "Could not seek to frame ", frame_number + 1, ": " "End of file reached" ); } - } while( this->frame_number() < frame_number ); + } while( mode == SEEK_MODE_EXACT && this->frame_number() < frame_number ); // Check for success - if( this->frame_number() == frame_number ) + if( ( mode == SEEK_MODE_EXACT && this->frame_number() == frame_number ) || + ( mode != SEEK_MODE_EXACT && frame && frame->frame->key_frame && + this->frame_number() <= frame_number ) ) { - break; + if( parent->klv_enabled && advance_count <= 1 && false ) + { + auto const chosen_frame_number = this->frame_number(); + converted_timestamp = + av_rescale_q( + frame_number - backstep - 1, av_inv_q( frame_rate() ), + video_stream->time_base ) + start_ts; + + clear_state_for_seek(); + throw_error_code( + av_seek_frame( + format_context.get(), video_stream->index, converted_timestamp, + AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY ), + "Could not seek to frame ", frame_number ); + if( codec_context ) + { + avcodec_flush_buffers( codec_context.get() ); + } + + advance_count = 0; + do + { + advance( advance_count == 0 ); + ++advance_count; + if( at_eof || this->frame_number() > chosen_frame_number ) + { + throw_error( + "Could not seek to frame ", frame_number + 1, ": " + "KLV re-seek failed" ); + } + } while( this->frame_number() < chosen_frame_number ); + + if( advance_count <= 1 ) + { + LOG_WARN( + parent->logger, + "KLV re-seek failed; " + "KLV reported for first frame may be incomplete" ); + } + } + + if( mode != SEEK_MODE_EXACT ) + { + auto& image_packets = frame->get_raw_image().packets; + for( auto it = image_packets.begin(); it != image_packets.end(); ) + { + if( !( ( *it )->flags & AV_PKT_FLAG_KEY ) ) + { + it = image_packets.erase( it ); + } + else + { + ++it; + } + } + } + + return; } } - // Check for failure - if( this->frame_number() > frame_number ) + // Backup slow strategy + if( !frame_count || *frame_count > frame_number || + mode == SEEK_MODE_KEYFRAME_BEFORE ) { - LOG_WARN( - kv::get_logger( "klv" ), - "Could not seek exactly to frame " << ( frame_number + 1 ) << ": " - "Ended up on frame " << ( this->frame_number() + 1 ) ); + seek_to_start(); + advance(); + } + + int64_t last_keyframe_pts = AV_NOPTS_VALUE; + int64_t last_keyframe_dts = AV_NOPTS_VALUE; + for( kv::frame_id_t i = *frame_count; i < frame_number; ++i ) + { + advance(); + + if( mode == SEEK_MODE_KEYFRAME_BEFORE && frame && frame->frame->key_frame ) + { + last_keyframe_dts = frame->get_raw_image().frame_dts; + last_keyframe_pts = frame->frame->pts; + } + + if( at_eof ) + { + throw_error( + "Could not seek to frame ", frame_number + 1, ": End of file reached" ); + } + } + + if( mode == SEEK_MODE_KEYFRAME_BEFORE ) + { + auto success = false; + for( auto const last_keyframe_ts : + { last_keyframe_pts, last_keyframe_dts } ) + { + if( last_keyframe_ts == AV_NOPTS_VALUE && frame_number > 0 ) + { + continue; + } + + clear_state_for_seek(); + throw_error_code( + av_seek_frame( + format_context.get(), video_stream->index, last_keyframe_ts, + AVSEEK_FLAG_BACKWARD ), + "Could not seek to frame ", frame_number + 1 ); + + if( codec_context ) + { + avcodec_flush_buffers( codec_context.get() ); + } + + do + { + advance(); + } while( + frame_number > 0 && frame && !frame->frame->key_frame && + frame->frame->pts < last_keyframe_pts ); + + if( frame_number <= 0 || ( frame && frame->frame->key_frame ) ) + { + success = true; + break; + } + } + + if( !success ) + { + throw_error( + "Could not seek to keyframe before frame ", frame_number + 1 ); + } + + frame_count.emplace( frame_number ); } } @@ -1495,9 +2126,13 @@ double ffmpeg_video_input::priv::open_video_state ::duration() const { - return - ( video_stream->start_time + video_stream->duration - start_ts ) * - av_q2d( video_stream->time_base ); + if( video_stream->start_time != AV_NOPTS_VALUE && video_stream->duration > 0 ) + { + return + ( video_stream->start_time + video_stream->duration - start_ts ) * + av_q2d( video_stream->time_base ); + } + return 0.0; } // ---------------------------------------------------------------------------- @@ -1505,12 +2140,11 @@ AVRational ffmpeg_video_input::priv::open_video_state ::frame_rate() const { - auto result = video_stream->avg_frame_rate; - if( !result.num ) + if( video_stream->avg_frame_rate.num ) { - result = video_stream->r_frame_rate; + return video_stream->avg_frame_rate; } - return result; + return maybe_frame_rate; } // ---------------------------------------------------------------------------- @@ -1518,6 +2152,11 @@ size_t ffmpeg_video_input::priv::open_video_state ::num_frames() const { + if( video_stream->nb_frames > 0 ) + { + return static_cast< size_t >( video_stream->nb_frames ); + } + return static_cast< size_t >( duration() * av_q2d( frame_rate() ) + 0.5 ); } @@ -1526,8 +2165,18 @@ kv::frame_id_t ffmpeg_video_input::priv::open_video_state ::frame_number() const { - if( !frame.has_value() || - frame->frame->best_effort_timestamp == AV_NOPTS_VALUE ) + if( !frame.has_value() ) + { + return -1; + } + + if( frame_count ) + { + return *frame_count; + } + + if( frame->frame->best_effort_timestamp == AV_NOPTS_VALUE || + frame_rate().num <= 0 ) { return -1; } @@ -1545,9 +2194,20 @@ ::timestamp() const { return {}; } - return kv::timestamp{ - static_cast< kv::time_usec_t >( curr_time() * 1000000.0 + 0.5 ), - frame_number() + 1 }; + + kv::timestamp ts; + if( frame->frame->best_effort_timestamp != AV_NOPTS_VALUE ) + { + ts.set_time_usec( + static_cast< kv::time_usec_t >( curr_time() * 1000000.0 + 0.5 ) ); + } + + if( frame_rate().num > 0 ) + { + ts.set_frame( frame_number() + 1 ); + } + + return ts; } // ---------------------------------------------------------------------------- @@ -1557,7 +2217,16 @@ ::implementation_settings() const { ffmpeg_video_settings_uptr result{ new ffmpeg_video_settings{} }; result->frame_rate = frame_rate(); - result->klv_stream_count = klv_streams.size(); + for( auto const& stream : klv_streams ) + { + result->klv_streams.emplace_back( stream.settings() ); + } + for( auto const& stream: audio_streams ) + { + result->audio_streams.emplace_back( stream.settings() ); + } + result->time_base = video_stream->time_base; + result->start_timestamp = format_context->start_time; if( codec_context ) { @@ -1592,7 +2261,8 @@ ::ffmpeg_video_input() set_capability( kva::video_input::HAS_TIMEOUT, false ); set_capability( kva::video_input::IS_SEEKABLE, true ); set_capability( kva::video_input::HAS_RAW_IMAGE, true ); - set_capability( kva::video_input::HAS_RAW_METADATA, false ); + set_capability( kva::video_input::HAS_RAW_METADATA, true ); + set_capability( kva::video_input::HAS_UNINTERPRETED_DATA, true ); ffmpeg_init(); } @@ -1621,18 +2291,19 @@ ::get_configuration() const "deinterlacing only to frames which are interlaced. " "See details at https://ffmpeg.org/ffmpeg-filters.html" ); - config->set_value( - "imagery_enabled", d->imagery_enabled, - "When set to false, will not attempt to process any imagery found in the " - "video file. This may be useful if only processing metadata." - ); - config->set_value( "klv_enabled", d->klv_enabled, "When set to false, will not attempt to process any KLV metadata found in " "the video file. This may be useful if only processing imagery." ); + config->set_value( + "audio_enabled", d->audio_enabled, + "When set to false, will not attempt to pass along any audio streams found " + "in the video file. This may be useful if no transcoding is to be done, or " + "if audio is to be dropped." + ); + config->set_value( "use_misp_timestamps", d->use_misp_timestamps, "When set to true, will attempt to use correlate KLV packet data to " @@ -1695,12 +2366,12 @@ ::set_configuration( kv::config_block_sptr in_config ) config->get_value< std::string >( "filter_desc", d->filter_description ); - d->imagery_enabled = - config->get_value< bool >( "imagery_enabled", d->imagery_enabled ); - d->klv_enabled = config->get_value< bool >( "klv_enabled", d->klv_enabled ); + d->audio_enabled = + config->get_value< bool >( "audio_enabled", d->audio_enabled ); + d->use_misp_timestamps = config->get_value< bool >( "use_misp_timestamps", d->use_misp_timestamps ); @@ -1801,6 +2472,16 @@ ffmpeg_video_input ::seek_frame( kv::timestamp& ts, kv::timestamp::frame_t frame_number, uint32_t timeout ) +{ + return seek_frame_( ts, frame_number, SEEK_MODE_EXACT, timeout ); +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input +::seek_frame_( + kv::timestamp& ts, kv::timestamp::frame_t frame_number, + ffmpeg_video_input::seek_mode mode, uint32_t timeout ) { d->assert_open( "seek_frame()" ); @@ -1820,7 +2501,7 @@ ::seek_frame( kv::timestamp& ts, try { - d->video->seek( frame_number - 1 ); + d->video->seek( frame_number - 1, mode ); ts = frame_timestamp(); return true; } @@ -1893,9 +2574,34 @@ ::raw_frame_metadata() return nullptr; } + for( auto& stream : d->video->klv_streams ) + { + for( auto& packet : stream.this_frame_buffer ) + { + ffmpeg_video_raw_metadata::packet_info info; + info.packet = std::move( packet ); + info.stream_settings = stream.settings(); + d->video->frame->get_raw_metadata().packets + .emplace_back( std::move( info ) ); + } + stream.this_frame_buffer.clear(); + } return d->video->frame->raw_metadata; } +// ---------------------------------------------------------------------------- +kv::video_uninterpreted_data_sptr +ffmpeg_video_input +::uninterpreted_frame_data() +{ + if( !d->is_valid() ) + { + return nullptr; + } + + return d->video->frame->uninterpreted_data; +} + // ---------------------------------------------------------------------------- kv::metadata_map_sptr ffmpeg_video_input @@ -1955,6 +2661,22 @@ ::num_frames() const return d->video->num_frames(); } +// ---------------------------------------------------------------------------- +double +ffmpeg_video_input +::frame_rate() +{ + d->assert_open( "frame_rate()" ); + + auto const result = d->video->frame_rate(); + if( result.num > 0 && result.den > 0 ) + { + return av_q2d( result ); + } + + return -1.0; +} + // ---------------------------------------------------------------------------- kv::video_settings_uptr ffmpeg_video_input diff --git a/arrows/ffmpeg/ffmpeg_video_input.h b/arrows/ffmpeg/ffmpeg_video_input.h index ffc59f9816..cfa3bf954b 100644 --- a/arrows/ffmpeg/ffmpeg_video_input.h +++ b/arrows/ffmpeg/ffmpeg_video_input.h @@ -3,45 +3,41 @@ // https://github.com/Kitware/kwiver/blob/master/LICENSE for details. /// \file -/// \brief \todo +/// Declaration of FFmpeg-based video input. -#ifndef KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_INPUT_H -#define KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_INPUT_H - -#include +#ifndef KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_INPUT_H_ +#define KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_INPUT_H_ #include #include + +#include + namespace kwiver { namespace arrows { namespace ffmpeg { -/// Video input using ffmpeg services. // ---------------------------------------------------------------------------- - -/// This class implements a video input algorithm using ffmpeg video services. -/// +/// Video input using FFmpeg (libav). class KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_input : public vital::algo::video_input { public: - /// Constructor + enum seek_mode { + SEEK_MODE_EXACT, + SEEK_MODE_KEYFRAME_BEFORE, + }; + ffmpeg_video_input(); virtual ~ffmpeg_video_input(); - PLUGIN_INFO( "ffmpeg", - "Use FFMPEG to read video files as a sequence of images." ) + PLUGIN_INFO( + "ffmpeg", "Use FFmpeg to read video files as a sequence of images." ) - /// Get this algorithm's \link vital::config_block configuration block - /// \endlink vital::config_block_sptr get_configuration() const override; - - /// Set this algorithm's properties via a config block void set_configuration( vital::config_block_sptr config ) override; - - /// Check that the algorithm's currently configuration is valid bool check_configuration( vital::config_block_sptr config ) const override; void open( std::string video_name ) override; @@ -52,24 +48,29 @@ class KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_input bool seekable() const override; size_t num_frames() const override; + double frame_rate() override; + + bool next_frame( vital::timestamp& ts, uint32_t timeout = 0 ) override; + bool seek_frame( + vital::timestamp& ts, vital::timestamp::frame_t frame_number, + uint32_t timeout = 0 ) override; - bool next_frame( ::kwiver::vital::timestamp& ts, - uint32_t timeout = 0 ) override; - bool seek_frame( ::kwiver::vital::timestamp& ts, - ::kwiver::vital::timestamp::frame_t frame_number, - uint32_t timeout = 0 ) override; + bool seek_frame_( + vital::timestamp& ts, vital::timestamp::frame_t frame_number, + seek_mode mode, uint32_t timeout = 0 ); - ::kwiver::vital::timestamp frame_timestamp() const override; - ::kwiver::vital::image_container_sptr frame_image() override; - ::kwiver::vital::video_raw_image_sptr raw_frame_image() override; - ::kwiver::vital::metadata_vector frame_metadata() override; - ::kwiver::vital::video_raw_metadata_sptr raw_frame_metadata() override; - ::kwiver::vital::metadata_map_sptr metadata_map() override; + vital::timestamp frame_timestamp() const override; + vital::image_container_sptr frame_image() override; + vital::video_raw_image_sptr raw_frame_image() override; + vital::metadata_vector frame_metadata() override; + vital::video_raw_metadata_sptr raw_frame_metadata() override; + vital::video_uninterpreted_data_sptr uninterpreted_frame_data() override; + vital::metadata_map_sptr metadata_map() override; - ::kwiver::vital::video_settings_uptr implementation_settings() const override; + vital::video_settings_uptr implementation_settings() const override; private: - /// private implementation class + /// Private implementation class class priv; const std::unique_ptr< priv > d; @@ -79,6 +80,6 @@ class KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_input } // namespace arrows -} // end namespace +} // namespace kwiver -#endif // KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_INPUT_H +#endif diff --git a/arrows/ffmpeg/ffmpeg_video_input_clip.cxx b/arrows/ffmpeg/ffmpeg_video_input_clip.cxx new file mode 100644 index 0000000000..aabfd70fb9 --- /dev/null +++ b/arrows/ffmpeg/ffmpeg_video_input_clip.cxx @@ -0,0 +1,438 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Definition of FFmpeg video clipping utility. + +#include + +#include +#include +#include +#include +#include + +#include + +namespace kwiver { + +namespace arrows { + +namespace ffmpeg { + +// ---------------------------------------------------------------------------- +class ffmpeg_video_input_clip::impl +{ +public: + impl(); + + void seek_to_start(); + void filter_metadata( + vital::metadata_vector& metadata, vital::timestamp const& ts ) const; + vital::frame_id_t true_frame_begin() const; + vital::frame_id_t true_frame_end() const; + + std::shared_ptr< ffmpeg_video_input > video; + vital::frame_id_t frame_begin; + vital::frame_id_t frame_end; + + vital::metadata_map_sptr all_metadata; + std::string video_name; + vital::timestamp initial_timestamp; + int64_t initial_pts; + bool start_at_keyframe; + bool before_first_frame; +}; + +// ---------------------------------------------------------------------------- +ffmpeg_video_input_clip::impl +::impl() + : video{ new ffmpeg_video_input }, + frame_begin{ 0 }, + frame_end{ 0 }, + all_metadata{ nullptr }, + video_name{}, + initial_timestamp{}, + initial_pts{ AV_NOPTS_VALUE }, + start_at_keyframe{ false }, + before_first_frame{ true } +{} + +// ---------------------------------------------------------------------------- +void +ffmpeg_video_input_clip::impl +::seek_to_start() +{ + if( !video->seek_frame_( + initial_timestamp, frame_begin, + start_at_keyframe + ? ffmpeg_video_input::SEEK_MODE_KEYFRAME_BEFORE + : ffmpeg_video_input::SEEK_MODE_EXACT ) ) + { + throw_error( "Could not start video clip" ); + } +} + +// ---------------------------------------------------------------------------- +void +ffmpeg_video_input_clip::impl +::filter_metadata( + vital::metadata_vector& metadata, vital::timestamp const& ts ) const +{ + for( auto& md : metadata ) + { + if( md ) + { + md.reset( new vital::metadata( *md ) ); + md->set_timestamp( ts ); + } + } +} + +// ---------------------------------------------------------------------------- +vital::frame_id_t +ffmpeg_video_input_clip::impl +::true_frame_begin() const +{ + return + initial_timestamp.has_valid_frame() + ? initial_timestamp.get_frame() + : frame_begin; +} + +// ---------------------------------------------------------------------------- +vital::frame_id_t +ffmpeg_video_input_clip::impl +::true_frame_end() const +{ + return + video->num_frames() + ? std::min< vital::frame_id_t >( frame_end, video->num_frames() ) + : frame_end; +} + +// ---------------------------------------------------------------------------- +ffmpeg_video_input_clip +::ffmpeg_video_input_clip() + : d{ new impl } +{ + attach_logger( "ffmpeg_video_input_clip" ); +} + +// ---------------------------------------------------------------------------- +ffmpeg_video_input_clip +::~ffmpeg_video_input_clip() +{} + +// ---------------------------------------------------------------------------- +vital::config_block_sptr +ffmpeg_video_input_clip +::get_configuration() const +{ + auto config = vital::algo::video_input::get_configuration(); + config->set_value( + "frame_begin", d->frame_begin, + "First frame to include in the clip. Indexed from 1." ); + config->set_value( + "frame_end", d->frame_end, + "First frame not to include in the clip, i.e. one past the final frame in " + "the clip. Indexed from 1." ); + config->set_value( + "start_at_keyframe", d->start_at_keyframe, + "Start at the first keyframe before frame_begin, if frame_begin is not a " + "keyframe." ); + + vital::algo::video_input:: + get_nested_algo_configuration( "video_input", config, d->video ); + return config; +} + +// ---------------------------------------------------------------------------- +void +ffmpeg_video_input_clip +::set_configuration( vital::config_block_sptr in_config ) +{ + auto config = get_configuration(); + config->merge_config( in_config ); + + d->frame_begin = + config->get_value< vital::frame_id_t >( "frame_begin", d->frame_begin ); + d->frame_end = + config->get_value< vital::frame_id_t >( "frame_end", d->frame_end ); + d->start_at_keyframe = + config->get_value< bool >( "start_at_keyframe", d->start_at_keyframe ); + + d->video->set_configuration( config->subblock_view( "video_input:ffmpeg" ) ); +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input_clip +::check_configuration( vital::config_block_sptr config ) const +{ + if( !config->has_value( "frame_begin" ) || + !config->has_value( "frame_end" ) || + !config->has_value( "video_input:type" ) ) + { + return false; + } + + auto const frame_begin = + config->get_value< vital::frame_id_t >( "frame_begin" ); + auto const frame_end = + config->get_value< vital::frame_id_t >( "frame_end" ); + + return frame_begin <= frame_end && frame_begin > 0; +} + +// ---------------------------------------------------------------------------- +void +ffmpeg_video_input_clip +::open( std::string video_name ) +{ + d->video_name = video_name; + d->before_first_frame = true; + d->video->open( video_name ); + d->seek_to_start(); + auto const raw_image = + dynamic_cast< ffmpeg_video_raw_image const* >( + d->video->raw_frame_image().get() ); + if( !raw_image || raw_image->frame_pts == AV_NOPTS_VALUE ) + { + throw std::runtime_error( "Could not acquire PTS of first frame" ); + } + d->initial_pts = raw_image->frame_pts; + + auto const& capabilities = d->video->get_implementation_capabilities(); + using vi = vital::algo::video_input; + for( auto const& capability : { + vi::HAS_EOV, + vi::HAS_FRAME_NUMBERS, + vi::HAS_FRAME_DATA, + vi::HAS_FRAME_TIME, + vi::HAS_METADATA, + vi::HAS_ABSOLUTE_FRAME_TIME, + vi::HAS_TIMEOUT, + vi::IS_SEEKABLE, + vi::HAS_RAW_IMAGE, + vi::HAS_RAW_METADATA, + vi::HAS_UNINTERPRETED_DATA, } ) + { + set_capability( capability, capabilities.capability( capability ) ); + } +} + +// ---------------------------------------------------------------------------- +void +ffmpeg_video_input_clip +::close() +{ + d->all_metadata.reset(); + d->video->close(); +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input_clip +::end_of_video() const +{ + if( d->before_first_frame ) + { + return false; + } + + return + d->video->end_of_video() || + ( d->video->frame_timestamp().get_frame() >= d->frame_end ); +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input_clip +::good() const +{ + if( d->before_first_frame || end_of_video() ) + { + return false; + } + + return d->video->good(); +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input_clip +::seekable() const +{ + return d->video->seekable(); +} + +// ---------------------------------------------------------------------------- +size_t +ffmpeg_video_input_clip +::num_frames() const +{ + return d->true_frame_end() - d->true_frame_begin(); +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input_clip +::next_frame( vital::timestamp& ts, uint32_t timeout ) +{ + if( end_of_video() ) + { + ts = vital::timestamp{}; + return false; + } + + if( d->before_first_frame ) + { + d->before_first_frame = false; + ts = frame_timestamp(); + return true; + } + + vital::timestamp tmp_ts; + auto const success = + d->video->next_frame( tmp_ts, timeout ) && !end_of_video(); + ts = success ? frame_timestamp() : vital::timestamp{}; + return success; +} + +// ---------------------------------------------------------------------------- +bool +ffmpeg_video_input_clip +::seek_frame( + vital::timestamp& ts, vital::timestamp::frame_t frame_number, + uint32_t timeout ) +{ + if( frame_number > 1 ) + { + frame_number += d->true_frame_begin(); + frame_number = std::min( frame_number, d->true_frame_end() ); + return d->video->seek_frame( ts, frame_number, timeout ); + } + else + { + d->seek_to_start(); + return good(); + } +} + +// ---------------------------------------------------------------------------- +vital::timestamp +ffmpeg_video_input_clip +::frame_timestamp() const +{ + auto video_ts = d->video->frame_timestamp(); + vital::timestamp ts; + if( video_ts.has_valid_frame() ) + { + ts.set_frame( video_ts.get_frame() - d->true_frame_begin() + 1 ); + } + if( video_ts.has_valid_time() && d->initial_timestamp.has_valid_time() ) + { + ts.set_time_usec( + video_ts.get_time_usec() - d->initial_timestamp.get_time_usec() ); + } + return ts; +} + +// ---------------------------------------------------------------------------- +vital::image_container_sptr +ffmpeg_video_input_clip +::frame_image() +{ + return d->before_first_frame ? nullptr : d->video->frame_image(); +} + +// ---------------------------------------------------------------------------- +vital::video_raw_image_sptr +ffmpeg_video_input_clip +::raw_frame_image() +{ + return d->before_first_frame ? nullptr : d->video->raw_frame_image(); +} + +// ---------------------------------------------------------------------------- +vital::metadata_vector +ffmpeg_video_input_clip +::frame_metadata() +{ + if( d->before_first_frame ) + { + return {}; + } + + auto result = d->video->frame_metadata(); + d->filter_metadata( result, frame_timestamp() ); + return result; +} + +// ---------------------------------------------------------------------------- +vital::video_raw_metadata_sptr +ffmpeg_video_input_clip +::raw_frame_metadata() +{ + return d->before_first_frame ? nullptr : d->video->raw_frame_metadata(); +} + +// ---------------------------------------------------------------------------- +vital::video_uninterpreted_data_sptr +ffmpeg_video_input_clip +::uninterpreted_frame_data() +{ + return d->before_first_frame ? nullptr : d->video->uninterpreted_frame_data(); +} + +// ---------------------------------------------------------------------------- +vital::metadata_map_sptr +ffmpeg_video_input_clip +::metadata_map() +{ + if( d->all_metadata ) + { + return d->all_metadata; + } + + ffmpeg_video_input_clip tmp_video; + tmp_video.set_configuration( get_configuration() ); + tmp_video.open( d->video_name ); + + vital::metadata_map::map_metadata_t result; + vital::timestamp ts; + while( tmp_video.next_frame( ts ) ) + { + result.emplace( ts.get_frame(), tmp_video.frame_metadata() ); + } + + d->all_metadata.reset( + new vital::simple_metadata_map{ std::move( result ) } ); + return d->all_metadata; +} + +// ---------------------------------------------------------------------------- +vital::video_settings_uptr +ffmpeg_video_input_clip +::implementation_settings() const +{ + auto const settings = d->video->implementation_settings(); + auto const ffmpeg_settings = + dynamic_cast< ffmpeg_video_settings const* >( settings.get() ); + if( !ffmpeg_settings ) + { + return nullptr; + } + + auto result = *ffmpeg_settings; + result.start_timestamp = d->initial_pts; + return std::make_unique< ffmpeg_video_settings >( std::move( result ) ); +} + +} // namespace ffmpeg + +} // namespace arrows + +} // end namespace diff --git a/arrows/ffmpeg/ffmpeg_video_input_clip.h b/arrows/ffmpeg/ffmpeg_video_input_clip.h new file mode 100644 index 0000000000..426cfdc446 --- /dev/null +++ b/arrows/ffmpeg/ffmpeg_video_input_clip.h @@ -0,0 +1,76 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Declaration of FFmpeg video clipping utility. + +#ifndef KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_INPUT_CLIP_H +#define KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_INPUT_CLIP_H + +#include +#include + +#include + +namespace kwiver { + +namespace arrows { + +namespace ffmpeg { + +// ---------------------------------------------------------------------------- +/// Video input which temporally clips an FFmpeg-sourced video. +/// +/// This implementation must have access to FFmpeg-level detailed information in +/// order to properly clip raw streams. +class KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_input_clip + : public vital::algo::video_input +{ +public: + ffmpeg_video_input_clip(); + virtual ~ffmpeg_video_input_clip(); + + PLUGIN_INFO( "ffmpeg_clip", "Clip an FFmpeg-sourced video." ) + + vital::config_block_sptr get_configuration() const override; + void set_configuration( vital::config_block_sptr config ) override; + bool check_configuration( vital::config_block_sptr config ) const override; + + void open( std::string video_name ) override; + void close() override; + + bool end_of_video() const override; + bool good() const override; + + bool seekable() const override; + size_t num_frames() const override; + + bool next_frame( vital::timestamp& ts, uint32_t timeout = 0 ) override; + bool seek_frame( + vital::timestamp& ts, vital::timestamp::frame_t frame_number, + uint32_t timeout = 0 ) override; + + vital::timestamp frame_timestamp() const override; + vital::image_container_sptr frame_image() override; + vital::video_raw_image_sptr raw_frame_image() override; + vital::metadata_vector frame_metadata() override; + vital::video_raw_metadata_sptr raw_frame_metadata() override; + vital::video_uninterpreted_data_sptr uninterpreted_frame_data() override; + vital::metadata_map_sptr metadata_map() override; + + vital::video_settings_uptr implementation_settings() const override; + +private: + class impl; + + std::unique_ptr< impl > const d; +}; + +} // namespace ffmpeg + +} // namespace arrows + +} // end namespace + +#endif diff --git a/arrows/ffmpeg/ffmpeg_video_output.cxx b/arrows/ffmpeg/ffmpeg_video_output.cxx index 10763377f7..35954f947e 100644 --- a/arrows/ffmpeg/ffmpeg_video_output.cxx +++ b/arrows/ffmpeg/ffmpeg_video_output.cxx @@ -10,6 +10,7 @@ #include "arrows/ffmpeg/ffmpeg_video_output.h" #include "arrows/ffmpeg/ffmpeg_video_raw_image.h" #include "arrows/ffmpeg/ffmpeg_video_settings.h" +#include "arrows/ffmpeg/ffmpeg_video_uninterpreted_data.h" extern "C" { #include @@ -18,6 +19,7 @@ extern "C" { #include } +#include #include namespace kv = kwiver::vital; @@ -28,6 +30,65 @@ namespace arrows { namespace ffmpeg { +namespace { + +// ---------------------------------------------------------------------------- +struct ffmpeg_audio_stream +{ + ffmpeg_audio_stream( + AVFormatContext* format_context, + ffmpeg_audio_stream_settings const& settings ); + + ffmpeg_audio_stream( ffmpeg_audio_stream const& ) = delete; + ffmpeg_audio_stream( ffmpeg_audio_stream&& ) = delete; + + ffmpeg_audio_stream_settings settings; + AVStream* stream; +}; + +// ---------------------------------------------------------------------------- +ffmpeg_audio_stream +::ffmpeg_audio_stream( + AVFormatContext* format_context, + ffmpeg_audio_stream_settings const& settings ) + : settings{ settings }, + stream{ nullptr } +{ + auto const codec = + throw_error_null( + avcodec_find_encoder( settings.parameters->codec_id ), + "Could not find audio codec for stream ", settings.index ); + + codec_context_uptr codec_context{ + throw_error_null( + avcodec_alloc_context3( codec ), "Could not allocate codec context" ) }; + + throw_error_code( + avcodec_parameters_to_context( + codec_context.get(), settings.parameters.get() ) ); + + codec_context->time_base = settings.time_base; + + throw_error_code( + avcodec_open2( codec_context.get(), codec, nullptr ), + "Could not open audio codec" + ); + + stream = + throw_error_null( + avformat_new_stream( format_context, codec ), + "Could not allocate audio stream" ); + + throw_error_code( + avcodec_parameters_copy( stream->codecpar, settings.parameters.get() ), + "Could not copy codec parameters" + ); + + stream->time_base = codec_context->time_base; +} + +} // namespace + // ---------------------------------------------------------------------------- class ffmpeg_video_output::impl { @@ -45,11 +106,13 @@ class ffmpeg_video_output::impl open_video_state& operator=( open_video_state&& ) = default; - bool try_codec( ffmpeg_video_settings const& settings ); + bool try_codec(); void add_image( kv::image_container_sptr const& image, kv::timestamp const& ts ); void add_image( vital::video_raw_image const& image ); + void add_uninterpreted_data( + vital::video_uninterpreted_data const& misc_data ); bool write_next_packet(); void write_remaining_packets(); @@ -65,11 +128,16 @@ class ffmpeg_video_output::impl #else AVOutputFormat* output_format; #endif + ffmpeg_video_settings video_settings; AVStream* video_stream; AVStream* metadata_stream; codec_context_uptr codec_context; AVCodec const* codec; sws_context_uptr image_conversion_context; + bsf_context_uptr annex_b_bsf; + int64_t prev_video_dts; + + std::list< ffmpeg_audio_stream > audio_streams; }; impl(); @@ -382,6 +450,15 @@ ::add_metadata( VITAL_UNUSED kwiver::vital::metadata const& md ) // TODO } +// ---------------------------------------------------------------------------- +void +ffmpeg_video_output +::add_uninterpreted_data( vital::video_uninterpreted_data const& misc_data ) +{ + d->assert_open( "add_image()" ); + d->video->add_uninterpreted_data( misc_data ); +} + // ---------------------------------------------------------------------------- vital::video_settings_uptr ffmpeg_video_output @@ -396,7 +473,13 @@ ::implementation_settings() const result->frame_rate = d->video->video_stream->avg_frame_rate; avcodec_parameters_from_context( result->parameters.get(), d->video->codec_context.get() ); - result->klv_stream_count = 0; // TODO + result->klv_streams = {}; + for( auto const& stream : d->video->audio_streams ) + { + result->audio_streams.emplace_back( stream.settings ); + } + result->time_base = d->video->video_stream->time_base; + result->start_timestamp = d->video->format_context->start_time; return kwiver::vital::video_settings_uptr{ result }; } @@ -409,11 +492,13 @@ ::open_video_state( frame_count{ 0 }, format_context{ nullptr }, output_format{ nullptr }, + video_settings{ settings }, video_stream{ nullptr }, metadata_stream{ nullptr }, codec_context{ nullptr }, codec{ nullptr }, - image_conversion_context{ nullptr } + image_conversion_context{ nullptr }, + prev_video_dts{ AV_NOPTS_VALUE } { // Allocate output format context { @@ -426,6 +511,9 @@ ::open_video_state( } output_format = format_context->oformat; + format_context->flags |= AVFMT_FLAG_AUTO_BSF; + format_context->flags |= AVFMT_FLAG_GENPTS; + // Prioritization scheme for codecs: // (1) Match ffmpeg settings passed to constructor if present // (2) Match configuration setting if present @@ -483,7 +571,7 @@ ::open_video_state( for( auto const possible_codec : possible_codecs ) { codec = possible_codec; - if( try_codec( settings ) ) + if( try_codec() ) { break; } @@ -503,20 +591,26 @@ ::open_video_state( av_dump_format( format_context.get(), video_stream->index, video_name.c_str(), 1 ); + for( auto const& stream_settings : settings.audio_streams ) + { + audio_streams.emplace_back( format_context.get(), stream_settings ); + av_dump_format( + format_context.get(), + audio_streams.back().stream->index, video_name.c_str(), 1 ); + } + // Open streams throw_error_code( avio_open( &format_context->pb, video_name.c_str(), AVIO_FLAG_WRITE ), "Could not open `", video_name, "` for writing" ); - auto const output_status = - avformat_init_output( format_context.get(), nullptr ); - if( output_status == AVSTREAM_INIT_IN_WRITE_HEADER ) - { - throw_error_code( - avformat_write_header( format_context.get(), nullptr ), - "Could not write video header" ); - } - throw_error_code( output_status, "Could not initialize output stream" ); + throw_error_code( + avformat_write_header( format_context.get(), nullptr ), + "Could not write video header" ); + + throw_error_code( + avformat_init_output( format_context.get(), nullptr ), + "Could not initialize output stream" ); } // ---------------------------------------------------------------------------- @@ -541,7 +635,7 @@ ffmpeg_video_output::impl::open_video_state // ---------------------------------------------------------------------------- bool ffmpeg_video_output::impl::open_video_state -::try_codec( ffmpeg_video_settings const& settings ) +::try_codec() { LOG_DEBUG( parent->logger, "Trying output codec: " << pretty_codec_name( codec ) ); @@ -551,20 +645,23 @@ ::try_codec( ffmpeg_video_settings const& settings ) throw_error_null( avcodec_alloc_context3( codec ), "Could not allocate codec context" ) ); + codec_context->thread_count = 0; + codec_context->thread_type = FF_THREAD_FRAME; + // Fill in fields from given settings - if( codec->id == settings.parameters->codec_id ) + if( codec->id == video_settings.parameters->codec_id ) { throw_error_code( avcodec_parameters_to_context( - codec_context.get(), settings.parameters.get() ) ); + codec_context.get(), video_settings.parameters.get() ) ); } else { - codec_context->width = settings.parameters->width; - codec_context->height = settings.parameters->height; + codec_context->width = video_settings.parameters->width; + codec_context->height = video_settings.parameters->height; } - codec_context->time_base = av_inv_q( settings.frame_rate ); - codec_context->framerate = settings.frame_rate; + codec_context->time_base = av_inv_q( video_settings.frame_rate ); + codec_context->framerate = video_settings.frame_rate; // Fill in backup parameters from config if( codec_context->pix_fmt < 0 ) @@ -617,7 +714,12 @@ ::try_codec( ffmpeg_video_settings const& settings ) video_stream->codecpar->height = codec_context->height; video_stream->codecpar->format = codec_context->pix_fmt; - auto const err = avcodec_open2( codec_context.get(), codec, nullptr ); + AVDictionary* codec_options = nullptr; + for( auto const& entry : video_settings.codec_options ) + { + av_dict_set( &codec_options, entry.first.c_str(), entry.second.c_str(), 0 ); + } + auto const err = avcodec_open2( codec_context.get(), codec, &codec_options ); if( err < 0 ) { LOG_WARN( @@ -664,24 +766,43 @@ ::add_image( kv::image_container_sptr const& image, // Give the frame the raw pixel data { - size_t index = 0; - auto ptr = static_cast< uint8_t* >( image->get_image().first_pixel() ); + auto ptr = + static_cast< uint8_t const* >( image->get_image().first_pixel() ); auto const i_step = image->get_image().h_step(); auto const j_step = image->get_image().w_step(); auto const k_step = image->get_image().d_step(); - for( size_t i = 0; i < image->height(); ++i ) + if( j_step == static_cast< ptrdiff_t >( image->depth() ) && + k_step == static_cast< ptrdiff_t >( 1 ) ) + { + for( size_t i = 0; i < image->height(); ++i ) + { + std::memcpy( + frame->data[ 0 ] + i * frame->linesize[ 0 ], ptr + i * i_step, + image->width() * image->depth() ); + } + } + else { - for( size_t j = 0; j < image->width(); ++j ) + auto const i_step_ptr = i_step - j_step * image->width(); + auto const j_step_ptr = j_step - k_step * image->depth(); + auto const k_step_ptr = k_step; + auto const i_step_index = + frame->linesize[ 0 ] - image->width() * image->depth(); + size_t index = 0; + for( size_t i = 0; i < image->height(); ++i ) { - for( size_t k = 0; k < image->depth(); ++k ) + for( size_t j = 0; j < image->width(); ++j ) { - frame->data[ 0 ][ index++ ] = *ptr; - ptr += k_step; + for( size_t k = 0; k < image->depth(); ++k ) + { + frame->data[ 0 ][ index++ ] = *ptr; + ptr += k_step_ptr; + } + ptr += j_step_ptr; } - ptr += j_step - k_step * image->depth(); + ptr += i_step_ptr; + index += i_step_index; } - ptr += i_step - j_step * image->width(); - index += frame->linesize[ 0 ] - image->width() * image->depth(); } } @@ -738,15 +859,120 @@ ::add_image( kv::video_raw_image const& image ) { auto const& ffmpeg_image = dynamic_cast< ffmpeg_video_raw_image const& >( image ); + + // Initialize bitstream filters + if( !annex_b_bsf && + ( codec_context->codec_id == AV_CODEC_ID_H264 || + codec_context->codec_id == AV_CODEC_ID_H265 ) ) + { + // Find filter + auto const bsf_name = + ( codec_context->codec_id == AV_CODEC_ID_H264 ) + ? "h264_mp4toannexb" + : "hevc_mp4toannexb"; + auto const bsf = av_bsf_get_by_name( bsf_name ); + + if( bsf ) + { + // Allocate filter context + AVBSFContext* bsf_context = nullptr; + av_bsf_alloc( bsf, &bsf_context ); + annex_b_bsf.reset( + throw_error_null( bsf_context, "Could not allocate BSF context" ) ); + + // Fill in filter parameters + throw_error_code( + avcodec_parameters_copy( + annex_b_bsf->par_in, video_settings.parameters.get() ), + "Could not copy codec parameters" ); + annex_b_bsf->time_base_in = video_settings.time_base; + + // Initialize filter + throw_error_code( + av_bsf_init( annex_b_bsf.get() ), + "Could not initialize Annex B filter" ); + } + } + for( auto const& packet : ffmpeg_image.packets ) { + // Ensure this packet has sensible timestamps or FFmpeg will complain + if( packet->pts == AV_NOPTS_VALUE || packet->dts == AV_NOPTS_VALUE || + packet->dts <= prev_video_dts || packet->dts > packet->pts ) + { + LOG_ERROR( + parent->logger, + "Dropping video packet with invalid dts/pts " + << packet->dts << "/" << packet->pts << " " + << "with prev dts " << prev_video_dts ); + continue; + } + + // Record this DTS for next time + prev_video_dts = packet->dts; + + // Copy the packet so we can switch the video stream index + packet_uptr tmp_packet{ + throw_error_null( + av_packet_clone( packet.get() ), "Could not copy video packet" ) }; + tmp_packet->stream_index = video_stream->index; + + // Convert MP4-compatible H.264/H.265 to TS-compatible + if( annex_b_bsf ) + { + throw_error_code( + av_bsf_send_packet( annex_b_bsf.get(), tmp_packet.get() ) ); + throw_error_code( + av_bsf_receive_packet( annex_b_bsf.get(), tmp_packet.get() ) ); + } + + av_packet_rescale_ts( + tmp_packet.get(), video_settings.time_base, video_stream->time_base ); + + // Write the packet throw_error_code( - av_interleaved_write_frame( format_context.get(), packet.get() ), + av_interleaved_write_frame( format_context.get(), tmp_packet.get() ), "Could not write frame to file" ); } ++frame_count; } +// ---------------------------------------------------------------------------- +void +ffmpeg_video_output::impl::open_video_state +::add_uninterpreted_data( vital::video_uninterpreted_data const& misc_data ) +{ + auto const& ffmpeg_data = + dynamic_cast< ffmpeg_video_uninterpreted_data const& >( misc_data ); + + for( auto const& packet : ffmpeg_data.audio_packets ) + { + for( auto const& stream : audio_streams ) + { + if( stream.settings.index != packet->stream_index ) + { + continue; + } + + // Copy the packet to switch the stream index + packet_uptr tmp_packet{ + throw_error_null( + av_packet_clone( packet.get() ), "Could not copy audio packet" ) }; + tmp_packet->stream_index = stream.stream->index; + + av_packet_rescale_ts( + tmp_packet.get(), stream.settings.time_base, stream.stream->time_base ); + + // Write the packet + throw_error_code( + av_interleaved_write_frame( format_context.get(), tmp_packet.get() ), + "Could not write frame to file" ); + + break; + } + } +} + // ---------------------------------------------------------------------------- bool ffmpeg_video_output::impl::open_video_state @@ -765,6 +991,18 @@ ::write_next_packet() } throw_error_code( err, "Could not get next packet from encoder" ); + // Adjust for any global timestamp offset + if( video_settings.start_timestamp != AV_NOPTS_VALUE ) + { + auto const offset = + av_rescale_q( + video_settings.start_timestamp, + AVRational{ 1, AV_TIME_BASE }, + video_stream->time_base ); + packet->dts += offset; + packet->pts += offset; + } + // Succeeded; write to file throw_error_code( av_interleaved_write_frame( format_context.get(), packet.get() ), diff --git a/arrows/ffmpeg/ffmpeg_video_output.h b/arrows/ffmpeg/ffmpeg_video_output.h index 9f4ff3a239..e3716a5175 100644 --- a/arrows/ffmpeg/ffmpeg_video_output.h +++ b/arrows/ffmpeg/ffmpeg_video_output.h @@ -57,6 +57,9 @@ class KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_output void add_metadata( vital::metadata const& md ) override; + void add_uninterpreted_data( + vital::video_uninterpreted_data const& misc_data ) override; + vital::video_settings_uptr implementation_settings() const override; private: diff --git a/arrows/ffmpeg/ffmpeg_video_raw_image.cxx b/arrows/ffmpeg/ffmpeg_video_raw_image.cxx index 5ce6644d7c..25680101c9 100644 --- a/arrows/ffmpeg/ffmpeg_video_raw_image.cxx +++ b/arrows/ffmpeg/ffmpeg_video_raw_image.cxx @@ -14,7 +14,11 @@ namespace arrows { namespace ffmpeg { ffmpeg_video_raw_image -::ffmpeg_video_raw_image() : packets{} +::ffmpeg_video_raw_image() + : packets{}, + frame_dts{ AV_NOPTS_VALUE }, + frame_pts{ AV_NOPTS_VALUE }, + is_keyframe{ true } {} } // namespace ffmpeg diff --git a/arrows/ffmpeg/ffmpeg_video_raw_image.h b/arrows/ffmpeg/ffmpeg_video_raw_image.h index 5734f3c876..f087ac0fd8 100644 --- a/arrows/ffmpeg/ffmpeg_video_raw_image.h +++ b/arrows/ffmpeg/ffmpeg_video_raw_image.h @@ -33,6 +33,9 @@ struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_raw_image operator=( ffmpeg_video_raw_image const& ) = delete; std::list< packet_uptr > packets; + int64_t frame_dts; + int64_t frame_pts; + bool is_keyframe; }; using ffmpeg_video_raw_image_sptr = std::shared_ptr< ffmpeg_video_raw_image >; diff --git a/arrows/ffmpeg/ffmpeg_video_raw_metadata.cxx b/arrows/ffmpeg/ffmpeg_video_raw_metadata.cxx index 92777b5d13..266f797573 100644 --- a/arrows/ffmpeg/ffmpeg_video_raw_metadata.cxx +++ b/arrows/ffmpeg/ffmpeg_video_raw_metadata.cxx @@ -13,10 +13,16 @@ namespace arrows { namespace ffmpeg { +// ---------------------------------------------------------------------------- ffmpeg_video_raw_metadata ::ffmpeg_video_raw_metadata() : packets{} {} +// ---------------------------------------------------------------------------- +ffmpeg_video_raw_metadata::packet_info +::packet_info() : packet{}, stream_settings{} +{} + } // namespace ffmpeg } // namespace arrows diff --git a/arrows/ffmpeg/ffmpeg_video_raw_metadata.h b/arrows/ffmpeg/ffmpeg_video_raw_metadata.h index b3ba3fee5c..e900e1c30f 100644 --- a/arrows/ffmpeg/ffmpeg_video_raw_metadata.h +++ b/arrows/ffmpeg/ffmpeg_video_raw_metadata.h @@ -11,6 +11,8 @@ #include #include +#include + #include #include @@ -31,7 +33,15 @@ struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_raw_metadata ffmpeg_video_raw_metadata& operator=( ffmpeg_video_raw_metadata const& ) = delete; - std::list< packet_uptr > packets; + struct packet_info + { + packet_info(); + + packet_uptr packet; + klv::klv_stream_settings stream_settings; + }; + + std::list< packet_info > packets; }; using ffmpeg_video_raw_metadata_sptr = std::shared_ptr< ffmpeg_video_raw_metadata >; diff --git a/arrows/ffmpeg/ffmpeg_video_settings.cxx b/arrows/ffmpeg/ffmpeg_video_settings.cxx index a1b450697d..a43934af31 100644 --- a/arrows/ffmpeg/ffmpeg_video_settings.cxx +++ b/arrows/ffmpeg/ffmpeg_video_settings.cxx @@ -20,7 +20,11 @@ ffmpeg_video_settings ::ffmpeg_video_settings() : frame_rate{ 0, 1 }, parameters{ avcodec_parameters_alloc() }, - klv_stream_count{ 0 } + klv_streams{}, + audio_streams{}, + time_base{ 0, 1 }, + start_timestamp{ AV_NOPTS_VALUE }, + codec_options{} { if( !parameters ) { @@ -34,7 +38,11 @@ ffmpeg_video_settings ::ffmpeg_video_settings( ffmpeg_video_settings const& other ) : frame_rate{ other.frame_rate }, parameters{ avcodec_parameters_alloc() }, - klv_stream_count{ other.klv_stream_count } + klv_streams{ other.klv_streams }, + audio_streams{ other.audio_streams }, + time_base{ other.time_base }, + start_timestamp{ other.start_timestamp }, + codec_options{ other.codec_options } { throw_error_code( avcodec_parameters_copy( parameters.get(), other.parameters.get() ), @@ -46,7 +54,11 @@ ffmpeg_video_settings ::ffmpeg_video_settings( ffmpeg_video_settings&& other ) : frame_rate{ std::move( other.frame_rate ) }, parameters{ std::move( other.parameters ) }, - klv_stream_count{ std::move( other.klv_stream_count ) } + klv_streams{ std::move( other.klv_streams ) }, + audio_streams{ std::move( other.audio_streams ) }, + time_base{ std::move( other.time_base ) }, + start_timestamp{ std::move( other.start_timestamp ) }, + codec_options{ std::move( other.codec_options ) } {} // ---------------------------------------------------------------------------- @@ -54,10 +66,13 @@ ffmpeg_video_settings ::ffmpeg_video_settings( size_t width, size_t height, AVRational frame_rate, - size_t klv_stream_count ) + std::vector< klv::klv_stream_settings > const& klv_streams ) : frame_rate( frame_rate ), parameters{ avcodec_parameters_alloc() }, - klv_stream_count{ klv_stream_count } + klv_streams{ klv_streams }, + time_base{ 0, 1 }, + start_timestamp{ AV_NOPTS_VALUE }, + codec_options{} { if( !parameters ) { @@ -83,7 +98,10 @@ ::operator=( ffmpeg_video_settings const& other ) throw_error_code( avcodec_parameters_copy( parameters.get(), other.parameters.get() ), "Could not copy codec parameters" ); - klv_stream_count = other.klv_stream_count; + klv_streams = other.klv_streams; + time_base = other.time_base; + start_timestamp = other.start_timestamp; + codec_options = other.codec_options; return *this; } @@ -94,7 +112,10 @@ ::operator=( ffmpeg_video_settings&& other ) { frame_rate = std::move( other.frame_rate ); parameters = std::move( other.parameters ); - klv_stream_count = std::move( other.klv_stream_count ); + klv_streams = std::move( other.klv_streams ); + time_base = std::move( other.time_base ); + start_timestamp = std::move( other.start_timestamp ); + codec_options = std::move( other.codec_options ); return *this; } diff --git a/arrows/ffmpeg/ffmpeg_video_settings.h b/arrows/ffmpeg/ffmpeg_video_settings.h index 2530f323ae..06075bebb8 100644 --- a/arrows/ffmpeg/ffmpeg_video_settings.h +++ b/arrows/ffmpeg/ffmpeg_video_settings.h @@ -9,15 +9,20 @@ #define KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_SETTINGS_H_ #include +#include #include +#include + #include extern "C" { #include } +#include #include +#include namespace kwiver { @@ -26,6 +31,11 @@ namespace arrows { namespace ffmpeg { // ---------------------------------------------------------------------------- +/// Parameters defining the desired characteristics of a video file. +/// +/// This struct will be filled in by ffmpeg_video_input when transcoding, or +/// by the user when created a new video from scratch. Members have been left +/// public so the user may modify them before passing to ffmpeg_video_output. struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_settings : public vital::video_settings { @@ -35,7 +45,7 @@ struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_settings ffmpeg_video_settings( size_t width, size_t height, AVRational frame_rate, - size_t klv_stream_count ); + std::vector< klv::klv_stream_settings > const& klv_streams = {} ); ~ffmpeg_video_settings(); @@ -44,9 +54,30 @@ struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_settings ffmpeg_video_settings& operator=( ffmpeg_video_settings&& other ); + /// Desired frame rate of the video. Must be set in most cases. AVRational frame_rate; + + /// FFmpeg's parameters determining how the video codec is set up. Notably, + /// height and width must be set before opening a video. codec_parameters_uptr parameters; - size_t klv_stream_count; + + /// Settings for each KLV stream to be inserted. + std::vector< klv::klv_stream_settings > klv_streams; + + /// Settings for each audio stream to be inserted. + std::vector< ffmpeg_audio_stream_settings > audio_streams; + + /// Time base of the video stream in the input video, if transcoding. Not + /// guaranteed to determine the time base in the output video. + AVRational time_base; + + /// Start time of the input video, in AV_TIME_BASE units (microseconds). + /// This information is necessary for copied and newly-encoded packets to sync + /// correctly. + int64_t start_timestamp; + + /// FFmpeg-defined string options passed to the video codec. + std::map< std::string, std::string > codec_options; }; using ffmpeg_video_settings_uptr = std::unique_ptr< ffmpeg_video_settings >; diff --git a/arrows/ffmpeg/ffmpeg_video_uninterpreted_data.cxx b/arrows/ffmpeg/ffmpeg_video_uninterpreted_data.cxx new file mode 100644 index 0000000000..4accfe6f78 --- /dev/null +++ b/arrows/ffmpeg/ffmpeg_video_uninterpreted_data.cxx @@ -0,0 +1,26 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Implementation of FFmpeg video uninterpreted data. + +#include + +namespace kwiver { + +namespace arrows { + +namespace ffmpeg { + +// ---------------------------------------------------------------------------- +ffmpeg_video_uninterpreted_data +::ffmpeg_video_uninterpreted_data() + : audio_packets{} +{} + +} // namespace ffmpeg + +} // namespace arrows + +} // namespace kwiver diff --git a/arrows/ffmpeg/ffmpeg_video_uninterpreted_data.h b/arrows/ffmpeg/ffmpeg_video_uninterpreted_data.h new file mode 100644 index 0000000000..15fc46058b --- /dev/null +++ b/arrows/ffmpeg/ffmpeg_video_uninterpreted_data.h @@ -0,0 +1,47 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Declaration of FFmpeg video uninterpreted data. + +#ifndef KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_UNINTERPRETED_DATA_H_ +#define KWIVER_ARROWS_FFMPEG_FFMPEG_VIDEO_UNINTERPRETED_DATA_H_ + +#include +#include + +#include + +#include +#include + +namespace kwiver { + +namespace arrows { + +namespace ffmpeg { + +// ---------------------------------------------------------------------------- +struct KWIVER_ALGO_FFMPEG_EXPORT ffmpeg_video_uninterpreted_data + : public vital::video_uninterpreted_data +{ + ffmpeg_video_uninterpreted_data(); + + ffmpeg_video_uninterpreted_data( + ffmpeg_video_uninterpreted_data const& ) = delete; + ffmpeg_video_uninterpreted_data& + operator=( ffmpeg_video_uninterpreted_data const& ) = delete; + + std::list< packet_uptr > audio_packets; +}; +using ffmpeg_video_uninterpreted_data_sptr = + std::shared_ptr< ffmpeg_video_uninterpreted_data >; + +} // namespace ffmpeg + +} // namespace arrows + +} // namespace kwiver + +#endif diff --git a/arrows/ffmpeg/register_algorithms.cxx b/arrows/ffmpeg/register_algorithms.cxx index bf4d84d8b8..bb7dd83ddf 100644 --- a/arrows/ffmpeg/register_algorithms.cxx +++ b/arrows/ffmpeg/register_algorithms.cxx @@ -9,6 +9,7 @@ #include #include +#include #include namespace kwiver { @@ -30,6 +31,7 @@ register_factories( kwiver::vital::plugin_loader& vpm ) } reg.register_algorithm< ::kwiver::arrows::ffmpeg::ffmpeg_video_input >(); + reg.register_algorithm< ::kwiver::arrows::ffmpeg::ffmpeg_video_input_clip >(); reg.register_algorithm< ::kwiver::arrows::ffmpeg::ffmpeg_video_output >(); reg.mark_module_as_loaded(); diff --git a/arrows/ffmpeg/tests/CMakeLists.txt b/arrows/ffmpeg/tests/CMakeLists.txt index e9c82f530c..2f1f41c001 100644 --- a/arrows/ffmpeg/tests/CMakeLists.txt +++ b/arrows/ffmpeg/tests/CMakeLists.txt @@ -9,8 +9,9 @@ set(test_libraries kwiver_algo_ffmpeg kwiver_algo_core) ############################## # Algorithms ffmpeg tests ############################## -kwiver_discover_gtests(ffmpeg video_input_ffmpeg LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") -kwiver_discover_gtests(ffmpeg video_output_ffmpeg LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") +kwiver_discover_gtests(ffmpeg video_input_ffmpeg LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") +kwiver_discover_gtests(ffmpeg video_input_ffmpeg_clip LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") +kwiver_discover_gtests(ffmpeg video_output_ffmpeg LIBRARIES ${test_libraries} ARGUMENTS "${kwiver_test_data_directory}") if( KWIVER_ENABLE_SERIALIZE_JSON ) kwiver_discover_gtests(ffmpeg video_input_ffmpeg_klv diff --git a/arrows/ffmpeg/tests/common.h b/arrows/ffmpeg/tests/common.h new file mode 100644 index 0000000000..e8d13fe415 --- /dev/null +++ b/arrows/ffmpeg/tests/common.h @@ -0,0 +1,145 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +#include + +#include +#include + +#include + +namespace ffmpeg = kwiver::arrows::ffmpeg; +namespace klv = kwiver::arrows::klv; +namespace kv = kwiver::vital; +namespace kvr = kwiver::vital::range; + +// ---------------------------------------------------------------------------- +// Verify the average difference between pixels is not too high. Some +// difference is expected due to compression artifacts, but we need to make +// sure the frame images we get out are generally the same as what we put in. +void +expect_eq_images( kv::image const& src_image, + kv::image const& tmp_image, + double epsilon ) +{ + auto error = 0.0; + + ASSERT_TRUE( src_image.width() == tmp_image.width() ); + ASSERT_TRUE( src_image.height() == tmp_image.height() ); + ASSERT_TRUE( src_image.depth() == tmp_image.depth() ); + + for( auto const i : kvr::iota( src_image.width() ) ) + { + for( auto const j : kvr::iota( src_image.height() ) ) + { + for( auto const k : kvr::iota( src_image.depth() ) ) + { + error += std::abs( + static_cast< double >( src_image.at< uint8_t >( i, j, k ) ) - + static_cast< double >( tmp_image.at< uint8_t >( i, j, k ) ) ); + } + } + } + error /= src_image.width() * src_image.height() * src_image.depth(); + + EXPECT_LE( error, epsilon ); +} + +// ---------------------------------------------------------------------------- +void +expect_eq_audio( kv::video_uninterpreted_data_sptr const& src_data, + kv::video_uninterpreted_data_sptr const& tmp_data ) +{ + ASSERT_EQ( src_data == nullptr, tmp_data == nullptr ); + if( !src_data ) + { + return; + } + + auto const& src_packets = + dynamic_cast< ffmpeg::ffmpeg_video_uninterpreted_data const& >( *src_data ) + .audio_packets; + auto const& tmp_packets = + dynamic_cast< ffmpeg::ffmpeg_video_uninterpreted_data const& >( *tmp_data ) + .audio_packets; + ASSERT_EQ( src_packets.size(), tmp_packets.size() ); + + auto src_it = src_packets.begin(); + auto tmp_it = tmp_packets.begin(); + while( src_it != src_packets.begin() && tmp_it != tmp_packets.begin() ) + { + ASSERT_EQ( ( *src_it )->size, ( *tmp_it )->size ); + EXPECT_TRUE( + std::equal( ( *src_it )->data, ( *src_it )->data + ( *src_it )->size, + ( *tmp_it )->data ) ); + ++src_it; + ++tmp_it; + } +} + +// ---------------------------------------------------------------------------- +void +expect_eq_videos( + kv::algo::video_input& src_is, kv::algo::video_input& tmp_is, + double image_epsilon = 0.0, kv::frame_id_t frame_offset = 0, + kv::time_usec_t usec_offset = 0, bool allow_different_lengths = false ) +{ + kv::timestamp src_ts; + kv::timestamp tmp_ts; + + // Check each pair of frames for equality + for( src_is.next_frame( src_ts ), tmp_is.next_frame( tmp_ts ); + !src_is.end_of_video() && !tmp_is.end_of_video(); + src_is.next_frame( src_ts ), tmp_is.next_frame( tmp_ts ) ) + { + SCOPED_TRACE( + std::string{ "Frame: " } + + std::to_string( src_ts.get_frame() ) + " | " + + std::to_string( tmp_ts.get_frame() ) ); + + EXPECT_EQ( src_ts.get_frame() + frame_offset, tmp_ts.get_frame() ); + EXPECT_NEAR( + src_ts.get_time_usec() + usec_offset, tmp_ts.get_time_usec(), 1 ); + + auto const src_data = src_is.uninterpreted_frame_data(); + auto const tmp_data = tmp_is.uninterpreted_frame_data(); + expect_eq_audio( src_data, tmp_data ); + + auto const src_image = src_is.frame_image()->get_image(); + auto const tmp_image = tmp_is.frame_image()->get_image(); + expect_eq_images( src_image, tmp_image, image_epsilon ); + } + if( !allow_different_lengths ) + { + EXPECT_TRUE( src_is.end_of_video() ); + EXPECT_TRUE( tmp_is.end_of_video() ); + } +} + +// ---------------------------------------------------------------------------- +void +expect_eq_videos( + std::string const& src_path, std::string const& tmp_path, + double image_epsilon = 0.0, kv::frame_id_t frame_offset = 0, + kv::time_usec_t usec_offset = 0, bool allow_different_lengths = false ) +{ + ASSERT_GE( frame_offset, 0 ); + ASSERT_GE( usec_offset, 0 ); + + ffmpeg::ffmpeg_video_input src_is; + ffmpeg::ffmpeg_video_input tmp_is; + src_is.open( src_path ); + tmp_is.open( tmp_path ); + + kv::timestamp ts; + for( kv::frame_id_t i = 0; i < frame_offset; ++i ) + { + src_is.next_frame( ts ); + } + + expect_eq_videos( src_is, tmp_is, image_epsilon ); + + src_is.close(); + tmp_is.close(); +} diff --git a/arrows/ffmpeg/tests/test_video_input_ffmpeg.cxx b/arrows/ffmpeg/tests/test_video_input_ffmpeg.cxx index be69d8d9e5..80bffde4e8 100644 --- a/arrows/ffmpeg/tests/test_video_input_ffmpeg.cxx +++ b/arrows/ffmpeg/tests/test_video_input_ffmpeg.cxx @@ -148,39 +148,6 @@ TEST_F ( ffmpeg_video_input, frame_image ) EXPECT_EQ( decode_barcode( *frame ), 1 ); } -// ---------------------------------------------------------------------------- -// Verify that disabling imagery processing acts as expected and doesn't break -// anything else. -TEST_F( ffmpeg_video_input, imagery_disabled ) -{ - ffmpeg::ffmpeg_video_input input; - - auto config = input.get_configuration(); - config->set_value< bool >( "imagery_enabled", false ); - input.set_configuration( config ); - input.open( aphill_video_path ); - - EXPECT_FALSE( input.good() ); - EXPECT_EQ( input.frame_image(), nullptr ); - - size_t frame_count = 0; - kv::timestamp ts; - while( input.next_frame( ts ) ) - { - ++frame_count; - EXPECT_TRUE( input.good() ); - EXPECT_EQ( input.frame_image(), nullptr ); - EXPECT_EQ( ts.get_frame(), frame_count ); - - auto const md = input.frame_metadata(); - ASSERT_FALSE( md.empty() ); - ASSERT_TRUE( md.at( 0 )->has( kv::VITAL_META_UNIX_TIMESTAMP ) ); - } - - input.close(); - EXPECT_FALSE( input.good() ); -} - // ---------------------------------------------------------------------------- // Verify that disabling KLV processing acts as expected and doesn't break // anything else. diff --git a/arrows/ffmpeg/tests/test_video_input_ffmpeg_clip.cxx b/arrows/ffmpeg/tests/test_video_input_ffmpeg_clip.cxx new file mode 100644 index 0000000000..ae7129a484 --- /dev/null +++ b/arrows/ffmpeg/tests/test_video_input_ffmpeg_clip.cxx @@ -0,0 +1,234 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Test the FFmpeg video clip input. + +#include + +#include + +#include +#include + +#include + +#include + +namespace ffmpeg = kwiver::arrows::ffmpeg; +namespace kv = kwiver::vital; + +std::filesystem::path g_data_dir; + +namespace { + +// ---------------------------------------------------------------------------- +void +configure_input( + ffmpeg::ffmpeg_video_input_clip& input, + kv::frame_id_t frame_begin, kv::frame_id_t frame_end, + bool start_at_keyframe ) +{ + auto config = input.get_configuration(); + config->set_value( "frame_begin", frame_begin ); + config->set_value( "frame_end", frame_end ); + config->set_value( "start_at_keyframe", start_at_keyframe ); + EXPECT_TRUE( input.check_configuration( config ) ); + input.set_configuration( config ); +} + +// ---------------------------------------------------------------------------- +void test_clipped( + ffmpeg::ffmpeg_video_input_clip& input, + std::filesystem::path const& filepath, + kv::frame_id_t frame_begin, kv::frame_id_t frame_end, + kv::time_usec_t usec_begin ) +{ + ffmpeg::ffmpeg_video_input unclipped_input; + unclipped_input.open( filepath.string() ); + kv::timestamp ts; + for( kv::frame_id_t i = 1; i < frame_begin; ++i ) + { + unclipped_input.next_frame( ts ); + } + + input.open( filepath.string() ); + EXPECT_FALSE( input.good() ); + EXPECT_FALSE( input.end_of_video() ); + ts = input.frame_timestamp(); + EXPECT_EQ( 1, ts.get_frame() ); + + CALL_TEST( + expect_eq_videos, + unclipped_input, input, 0.0, -frame_begin + 1, -usec_begin, true ); + + EXPECT_FALSE( input.good() ); + EXPECT_TRUE( input.end_of_video() ); + + if( !unclipped_input.end_of_video() ) + { + ts = unclipped_input.frame_timestamp(); + EXPECT_EQ( frame_end, ts.get_frame() ); + } + + unclipped_input.close(); + input.close(); + + EXPECT_FALSE( input.good() ); +} + +} // namespace + +// ---------------------------------------------------------------------------- +int +main( int argc, char* argv[] ) +{ + ::testing::InitGoogleTest( &argc, argv ); + TEST_LOAD_PLUGINS(); + + GET_ARG( 1, g_data_dir ); + + return RUN_ALL_TESTS(); +} + +// ---------------------------------------------------------------------------- +class ffmpeg_video_input_clip : public ::testing::Test +{ +public: + void SetUp() override + { + ffmpeg_video_path = data_dir / "videos/ffmpeg_video.mp4"; + aphill_video_path = data_dir / "videos/aphill_short.ts"; + } + + std::filesystem::path ffmpeg_video_path; + std::filesystem::path aphill_video_path; + TEST_ARG( data_dir ); +}; + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, create ) +{ + EXPECT_NE( nullptr, kv::algo::video_input::create( "ffmpeg_clip" ) ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, entire_video_exact_aphill ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 1, 49, false ); + CALL_TEST( test_clipped, input, aphill_video_path, 1, 49, 0 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, entire_video_keyframe_aphill ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 1, 49, true ); + CALL_TEST( test_clipped, input, aphill_video_path, 1, 49, 0 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, entire_video_exact_ffmpeg ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 1, 51, false ); + CALL_TEST( test_clipped, input, ffmpeg_video_path, 1, 51, 0 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, entire_video_keyframe_ffmpeg ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 1, 51, true ); + CALL_TEST( test_clipped, input, ffmpeg_video_path, 1, 51, 0 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, end_past_end ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 1, 100, false ); + CALL_TEST( test_clipped, input, aphill_video_path, 1, 49, 0 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, begin_past_end ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 100, 200, false ); + EXPECT_THROW( input.open( aphill_video_path.string() ), std::runtime_error ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, single_frame ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 20, 21, false ); + CALL_TEST( test_clipped, input, aphill_video_path, 20, 21, 633'966 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, non_keyframe_exact_aphill ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 7, 23, false ); + CALL_TEST( test_clipped, input, aphill_video_path, 7, 23, 200'200 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, non_keyframe_keyframe_aphill ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 7, 23, true ); + CALL_TEST( test_clipped, input, aphill_video_path, 1, 23, 0 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, non_keyframe_exact_ffmpeg ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 7, 23, false ); + CALL_TEST( test_clipped, input, ffmpeg_video_path, 7, 23, 1'200'000 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, non_keyframe_keyframe_ffmpeg ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 7, 23, true ); + CALL_TEST( test_clipped, input, ffmpeg_video_path, 6, 23, 1'000'000 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, keyframe_exact_aphill ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 17, 33, false ); + CALL_TEST( test_clipped, input, aphill_video_path, 17, 33, 533'866 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, keyframe_keyframe_aphill ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 17, 33, true ); + CALL_TEST( test_clipped, input, aphill_video_path, 17, 33, 533'866 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, keyframe_exact_ffmpeg ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 11, 33, false ); + CALL_TEST( test_clipped, input, ffmpeg_video_path, 11, 33, 2'000'000 ); +} + +// ---------------------------------------------------------------------------- +TEST_F ( ffmpeg_video_input_clip, keyframe_keyframe_ffmpeg ) +{ + ffmpeg::ffmpeg_video_input_clip input; + configure_input( input, 11, 33, true ); + CALL_TEST( test_clipped, input, ffmpeg_video_path, 11, 33, 2'000'000 ); +} diff --git a/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx b/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx index f9fcf04997..5b5c1a720d 100644 --- a/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx +++ b/arrows/ffmpeg/tests/test_video_output_ffmpeg.cxx @@ -5,23 +5,21 @@ #include #include +#include + #include #include +#include #include #include -#include #include -namespace ffmpeg = kwiver::arrows::ffmpeg; -namespace klv = kwiver::arrows::klv; -namespace kv = kwiver::vital; -namespace kvr = kwiver::vital::range; - kv::path_t g_data_dir; static std::string short_video_name = "videos/aphill_short.ts"; +static std::string audio_video_name = "videos/h264_audio.ts"; // ---------------------------------------------------------------------------- int @@ -109,68 +107,6 @@ class ffmpeg_video_output : public ::testing::Test TEST_ARG( data_dir ); }; -// ---------------------------------------------------------------------------- -// Verify the average difference between pixels is not too high. Some -// difference is expected due to compression artifacts, but we need to make -// sure the frame images we get out are generally the same as what we put in. -void -expect_eq_images( kv::image const& src_image, - kv::image const& tmp_image, - double epsilon ) -{ - auto error = 0.0; - - ASSERT_TRUE( src_image.width() == tmp_image.width() ); - ASSERT_TRUE( src_image.height() == tmp_image.height() ); - ASSERT_TRUE( src_image.depth() == tmp_image.depth() ); - - for( auto const i : kvr::iota( src_image.width() ) ) - { - for( auto const j : kvr::iota( src_image.height() ) ) - { - for( auto const k : kvr::iota( src_image.depth() ) ) - { - error += std::abs( - static_cast< double >( src_image.at< uint8_t >( i, j, k ) ) - - static_cast< double >( tmp_image.at< uint8_t >( i, j, k ) ) ); - } - } - } - error /= src_image.width() * src_image.height() * src_image.depth(); - - EXPECT_LE( error, epsilon ); -} - -// ---------------------------------------------------------------------------- -void -expect_eq_videos( std::string const& src_path, std::string const& tmp_path, - double image_epsilon ) -{ - ffmpeg::ffmpeg_video_input src_is; - ffmpeg::ffmpeg_video_input tmp_is; - kv::timestamp src_ts; - kv::timestamp tmp_ts; - src_is.open( src_path ); - tmp_is.open( tmp_path ); - - // Check each pair of frames for equality - for( src_is.next_frame( src_ts ), tmp_is.next_frame( tmp_ts ); - !src_is.end_of_video() && !tmp_is.end_of_video(); - src_is.next_frame( src_ts ), tmp_is.next_frame( tmp_ts ) ) - { - EXPECT_EQ( src_ts.get_frame(), tmp_ts.get_frame() ); - EXPECT_EQ( src_ts.get_time_usec(), tmp_ts.get_time_usec() ); - - auto const src_image = src_is.frame_image()->get_image(); - auto const tmp_image = tmp_is.frame_image()->get_image(); - expect_eq_images( src_image, tmp_image, image_epsilon ); - } - EXPECT_TRUE( src_is.end_of_video() ); - EXPECT_TRUE( tmp_is.end_of_video() ); - src_is.close(); - tmp_is.close(); -} - namespace { // This will delete the temporary file even if an exception is thrown @@ -223,9 +159,11 @@ TEST_F ( ffmpeg_video_output, round_trip ) } // Read the temporary file back in - expect_eq_videos( src_path, tmp_path, image_epsilon ); + CALL_TEST( expect_eq_videos, src_path, tmp_path, image_epsilon ); } +// ---------------------------------------------------------------------------- +// Similar to round_trip, but copying the video stream instead of re-encoding. TEST_F ( ffmpeg_video_output, round_trip_direct ) { auto const src_path = data_dir + "/" + short_video_name; @@ -273,7 +211,108 @@ TEST_F ( ffmpeg_video_output, round_trip_direct ) auto const image_epsilon = 0.0; // Read the temporary file back in - expect_eq_videos( src_path, tmp_path, image_epsilon ); + CALL_TEST( expect_eq_videos, src_path, tmp_path, image_epsilon ); +} + +// ---------------------------------------------------------------------------- +// Similar to round_trip, but for a test video with an audio stream. +TEST_F ( ffmpeg_video_output, round_trip_audio ) +{ + auto const src_path = data_dir + "/" + audio_video_name; + auto const tmp_path = + kwiver::testing::temp_file_name( "test-ffmpeg-output-", ".ts" ); + + kv::timestamp ts; + ffmpeg::ffmpeg_video_input is; + is.open( src_path ); + + ffmpeg::ffmpeg_video_output os; + os.open( tmp_path, is.implementation_settings().get() ); + _tmp_file_deleter tmp_file_deleter{ tmp_path }; + + // Write to a temporary file + for( is.next_frame( ts ); !is.end_of_video(); is.next_frame( ts ) ) + { + auto const image = is.frame_image(); + auto const uninterpreted_data = is.uninterpreted_frame_data(); + if( uninterpreted_data ) + { + os.add_uninterpreted_data( *uninterpreted_data ); + } + os.add_image( image, ts ); + } + os.close(); + is.close(); + + // Determined experimentally. 6.5 / 256 is non-negligable compression, but + // you can still see what the image is supposed to be + auto image_epsilon = 6.5; + + // Hardware decoding produces a lower-quality image + if( is.get_configuration()->get_value< bool >( "cuda_enabled", false ) ) + { + image_epsilon = 10.5; + } + + // Read the temporary file back in + CALL_TEST( expect_eq_videos, src_path, tmp_path, image_epsilon ); +} + +// ---------------------------------------------------------------------------- +// Similar to round_trip_direct, but for a test video with an audio stream. +TEST_F ( ffmpeg_video_output, round_trip_audio_direct ) +{ + auto const src_path = data_dir + "/" + audio_video_name; + auto const tmp_path = + kwiver::testing::temp_file_name( "test-ffmpeg-output-", ".ts" ); + + kv::timestamp ts; + ffmpeg::ffmpeg_video_input is; + is.open( src_path ); + + ffmpeg::ffmpeg_video_output os; + os.open( tmp_path, is.implementation_settings().get() ); + _tmp_file_deleter tmp_file_deleter{ tmp_path }; + + // Skip this test if we can't write the output video in the same format as + // the input video + { + auto const src_generic_settings = is.implementation_settings(); + auto const src_settings = + dynamic_cast< ffmpeg::ffmpeg_video_settings const* >( + src_generic_settings.get() ); + auto const tmp_generic_settings = os.implementation_settings(); + auto const tmp_settings = + dynamic_cast< ffmpeg::ffmpeg_video_settings const* >( + tmp_generic_settings.get() ); + if( !src_settings || !tmp_settings || + src_settings->parameters->codec_id != + tmp_settings->parameters->codec_id ) + { + return; + } + } + + // Write to a temporary file + for( is.next_frame( ts ); !is.end_of_video(); is.next_frame( ts ) ) + { + auto const image = is.raw_frame_image(); + ASSERT_TRUE( image ); + auto const uninterpreted_data = is.uninterpreted_frame_data(); + if( uninterpreted_data ) + { + os.add_uninterpreted_data( *uninterpreted_data ); + } + os.add_image( *image ); + } + os.close(); + is.close(); + + // Images should be identical + auto const image_epsilon = 0.0; + + // Read the temporary file back in + CALL_TEST( expect_eq_videos, src_path, tmp_path, image_epsilon ); } // ---------------------------------------------------------------------------- diff --git a/arrows/klv/CMakeLists.txt b/arrows/klv/CMakeLists.txt index 4acaadc04b..faf55a3517 100644 --- a/arrows/klv/CMakeLists.txt +++ b/arrows/klv/CMakeLists.txt @@ -17,6 +17,7 @@ set( sources klv_packet.cxx klv_read_write.cxx klv_set.cxx + klv_stream_settings.cxx klv_string.cxx klv_tag_traits.cxx klv_timeline.cxx @@ -77,6 +78,7 @@ set( public_headers klv_read_write.h klv_series.h klv_set.h + klv_stream_settings.h klv_string.h klv_timeline.h klv_types.h diff --git a/arrows/klv/applets/compare_klv.cxx b/arrows/klv/applets/compare_klv.cxx index cbb9a052ef..9bdf2554bd 100644 --- a/arrows/klv/applets/compare_klv.cxx +++ b/arrows/klv/applets/compare_klv.cxx @@ -107,7 +107,20 @@ class video_source : public vital::metadata_istream exit( EXIT_FAILURE ); } - m_video->open( filepath.string() ); + try + { + m_video->open( filepath.string() ); + } + catch( vital::video_runtime_exception const& e ) + { + std::cerr << e.what() << std::endl; + exit( EXIT_FAILURE ); + } + catch( vital::file_not_found_exception const& e ) + { + std::cerr << e.what() << std::endl; + exit( EXIT_FAILURE ); + } m_is.emplace( *m_video ); } @@ -661,10 +674,22 @@ ::run() // Loop through frames while( !lhs_is->at_end() || !rhs_is->at_end() ) { + // Determine frame numbers + auto const lhs_frame_number = + lhs_is->at_end() ? INT64_MAX : lhs_is->frame_number(); + auto const rhs_frame_number = + rhs_is->at_end() ? INT64_MAX : rhs_is->frame_number(); + auto const frame_number = std::min( lhs_frame_number, rhs_frame_number ); + d->breadcrumbs.emplace_back( + std::string{} + "frame (" + std::to_string( frame_number ) + ")" ); + // Extract information about this frame's KLV auto const build_data = - []( vital::metadata_istream& is ) -> std::vector< istream_data > { - if( is.at_end() ) + [ frame_number ]( + vital::metadata_istream& is, vital::frame_id_t this_frame_number ) + -> std::vector< istream_data > + { + if( is.at_end() || this_frame_number > frame_number ) { return {}; } @@ -676,17 +701,8 @@ ::run() } return data; }; - auto lhs_data = build_data( *lhs_is ); - auto rhs_data = build_data( *rhs_is ); - - // Determine frame numbers - auto const lhs_frame_number = - lhs_is->at_end() ? 0 : lhs_is->frame_number(); - auto const rhs_frame_number = - rhs_is->at_end() ? 0 : rhs_is->frame_number(); - auto const frame_number = std::max( lhs_frame_number, rhs_frame_number ); - d->breadcrumbs.emplace_back( - std::string{} + "frame (" + std::to_string( frame_number ) + ")" ); + auto lhs_data = build_data( *lhs_is, lhs_frame_number ); + auto rhs_data = build_data( *rhs_is, rhs_frame_number ); // Score each possible pair of packets on their similarity std::multimap< std::vector< size_t >, possible_pair > ranked_pairs; @@ -820,11 +836,11 @@ ::run() // Next frame d->breadcrumbs.pop_back(); - if( !lhs_is->at_end() ) + if( !lhs_is->at_end() && lhs_frame_number == frame_number ) { lhs_is->next_frame(); } - if( !rhs_is->at_end() ) + if( !rhs_is->at_end() && rhs_frame_number == frame_number ) { rhs_is->next_frame(); } diff --git a/arrows/klv/klv_0601.cxx b/arrows/klv/klv_0601.cxx index 0b5dd90293..1c16e4ed07 100644 --- a/arrows/klv/klv_0601.cxx +++ b/arrows/klv/klv_0601.cxx @@ -2820,7 +2820,7 @@ ::length_of_typed( klv_0601_wavelength_record const& item ) const // ---------------------------------------------------------------------------- klv_checksum_packet_format const* klv_0601_local_set_format -::checksum_format() const +::packet_checksum_format() const { return &m_checksum_format; } diff --git a/arrows/klv/klv_0601.h b/arrows/klv/klv_0601.h index e3d0609d1f..aa49de1d45 100644 --- a/arrows/klv/klv_0601.h +++ b/arrows/klv/klv_0601.h @@ -1042,7 +1042,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_0601_local_set_format description_() const override; klv_checksum_packet_format const* - checksum_format() const override; + packet_checksum_format() const override; private: klv_running_sum_16_packet_format m_checksum_format; diff --git a/arrows/klv/klv_0806.cxx b/arrows/klv/klv_0806.cxx index 0c155f32a6..b0dd441989 100644 --- a/arrows/klv/klv_0806.cxx +++ b/arrows/klv/klv_0806.cxx @@ -199,7 +199,7 @@ ::description_() const // ---------------------------------------------------------------------------- klv_checksum_packet_format const* klv_0806_local_set_format -::checksum_format() const +::packet_checksum_format() const { return &m_checksum_format; } diff --git a/arrows/klv/klv_0806.h b/arrows/klv/klv_0806.h index 4022dda879..c63793194f 100644 --- a/arrows/klv/klv_0806.h +++ b/arrows/klv/klv_0806.h @@ -66,7 +66,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_0806_local_set_format description_() const override; klv_checksum_packet_format const* - checksum_format() const override; + packet_checksum_format() const override; private: klv_crc_32_mpeg_packet_format m_checksum_format; diff --git a/arrows/klv/klv_0903.cxx b/arrows/klv/klv_0903.cxx index a455cfd799..ae9c9cdafd 100644 --- a/arrows/klv/klv_0903.cxx +++ b/arrows/klv/klv_0903.cxx @@ -174,7 +174,7 @@ ::description_() const // ---------------------------------------------------------------------------- klv_checksum_packet_format const* klv_0903_local_set_format -::checksum_format() const +::packet_checksum_format() const { return &m_checksum_format; } diff --git a/arrows/klv/klv_0903.h b/arrows/klv/klv_0903.h index 6c7fd6a44a..d2e1d17ef5 100644 --- a/arrows/klv/klv_0903.h +++ b/arrows/klv/klv_0903.h @@ -73,7 +73,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_0903_local_set_format description_() const override; klv_checksum_packet_format const* - checksum_format() const override; + packet_checksum_format() const override; private: klv_running_sum_16_packet_format m_checksum_format; diff --git a/arrows/klv/klv_1002.cxx b/arrows/klv/klv_1002.cxx index ea0f57a933..6484fe8cef 100644 --- a/arrows/klv/klv_1002.cxx +++ b/arrows/klv/klv_1002.cxx @@ -269,7 +269,7 @@ ::description_() const // ---------------------------------------------------------------------------- klv_checksum_packet_format const* klv_1002_local_set_format -::checksum_format() const +::packet_checksum_format() const { return &m_checksum_format; } diff --git a/arrows/klv/klv_1002.h b/arrows/klv/klv_1002.h index 9c232407da..aeb193f069 100644 --- a/arrows/klv/klv_1002.h +++ b/arrows/klv/klv_1002.h @@ -180,7 +180,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_1002_local_set_format description_() const override; klv_checksum_packet_format const* - checksum_format() const override; + packet_checksum_format() const override; private: klv_crc_16_ccitt_packet_format m_checksum_format; diff --git a/arrows/klv/klv_1107.cxx b/arrows/klv/klv_1107.cxx index ba36c13e1f..2a875068bf 100644 --- a/arrows/klv/klv_1107.cxx +++ b/arrows/klv/klv_1107.cxx @@ -113,7 +113,7 @@ ::klv_1107_local_set_format() // ---------------------------------------------------------------------------- klv_checksum_packet_format const* klv_1107_local_set_format -::checksum_format() const +::packet_checksum_format() const { return &m_checksum_format; } diff --git a/arrows/klv/klv_1107.h b/arrows/klv/klv_1107.h index 2c9f142416..922c6b1b34 100644 --- a/arrows/klv/klv_1107.h +++ b/arrows/klv/klv_1107.h @@ -109,7 +109,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_1107_local_set_format description_() const override; klv_checksum_packet_format const* - checksum_format() const override; + packet_checksum_format() const override; private: klv_crc_16_ccitt_packet_format m_checksum_format; diff --git a/arrows/klv/klv_1108.cxx b/arrows/klv/klv_1108.cxx index e2a6ecc874..7c9b3e4031 100644 --- a/arrows/klv/klv_1108.cxx +++ b/arrows/klv/klv_1108.cxx @@ -223,7 +223,7 @@ ::klv_1108_local_set_format() // ---------------------------------------------------------------------------- klv_checksum_packet_format const* klv_1108_local_set_format -::checksum_format() const +::packet_checksum_format() const { return &m_checksum_format; } diff --git a/arrows/klv/klv_1108.h b/arrows/klv/klv_1108.h index 603921dcac..29f4cf06c6 100644 --- a/arrows/klv/klv_1108.h +++ b/arrows/klv/klv_1108.h @@ -204,7 +204,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_1108_local_set_format description_() const override; klv_checksum_packet_format const* - checksum_format() const override; + packet_checksum_format() const override; private: klv_crc_16_ccitt_packet_format m_checksum_format; diff --git a/arrows/klv/klv_1303.hpp b/arrows/klv/klv_1303.hpp index 557b0690e1..69baed1092 100644 --- a/arrows/klv/klv_1303.hpp +++ b/arrows/klv/klv_1303.hpp @@ -434,7 +434,7 @@ ::read_typed( klv_read_iter_t& data, size_t length ) const // Array processing algorithm parameters std::unique_ptr< klv_data_format > format; uint64_t uint_bias; - element_t rle_default_element; + element_t rle_default_element = {}; switch( result.apa ) { case KLV_1303_APA_IMAP: diff --git a/arrows/klv/klv_data_format.cxx b/arrows/klv/klv_data_format.cxx index de944467a9..6aeffffdbf 100644 --- a/arrows/klv/klv_data_format.cxx +++ b/arrows/klv/klv_data_format.cxx @@ -61,7 +61,23 @@ ::to_string( klv_value const& value ) const // ---------------------------------------------------------------------------- klv_checksum_packet_format const* klv_data_format -::checksum_format() const +::prefix_checksum_format() const +{ + return nullptr; +} + +// ---------------------------------------------------------------------------- +klv_checksum_packet_format const* +klv_data_format +::payload_checksum_format() const +{ + return nullptr; +} + +// ---------------------------------------------------------------------------- +klv_checksum_packet_format const* +klv_data_format +::packet_checksum_format() const { return nullptr; } @@ -238,7 +254,8 @@ size_t klv_uint_format ::length_of_typed( uint64_t const& value ) const { - return m_length_constraints.fixed_or( klv_int_length( value ) ); + auto const int_length = klv_int_length( value ); + return std::max( m_length_constraints.fixed_or( 1 ), int_length ); } // ---------------------------------------------------------------------------- @@ -277,7 +294,8 @@ size_t klv_sint_format ::length_of_typed( int64_t const& value ) const { - return m_length_constraints.fixed_or( klv_int_length( value ) ); + auto const int_length = klv_int_length( value ); + return std::max( m_length_constraints.fixed_or( 1 ), int_length ); } // ---------------------------------------------------------------------------- @@ -392,7 +410,10 @@ size_t klv_float_format ::length_of_typed( klv_lengthy< double > const& value ) const { - return m_length_constraints.fixed_or( value.length ); + return + value.length + ? value.length + : m_length_constraints.fixed_or( m_length_constraints.suggested() ); } // ---------------------------------------------------------------------------- @@ -449,7 +470,10 @@ size_t klv_sflint_format ::length_of_typed( klv_lengthy< double > const& value ) const { - return m_length_constraints.fixed_or( value.length ); + return + value.length + ? value.length + : m_length_constraints.fixed_or( m_length_constraints.suggested() ); } // ---------------------------------------------------------------------------- @@ -517,7 +541,10 @@ size_t klv_uflint_format ::length_of_typed( klv_lengthy< double > const& value ) const { - return m_length_constraints.fixed_or( value.length ); + return + value.length + ? value.length + : m_length_constraints.fixed_or( m_length_constraints.suggested() ); } // ---------------------------------------------------------------------------- @@ -584,7 +611,10 @@ size_t klv_imap_format ::length_of_typed( klv_lengthy< double > const& value ) const { - return m_length_constraints.fixed_or( value.length ); + return + value.length + ? value.length + : m_length_constraints.fixed_or( m_length_constraints.suggested() ); } // ---------------------------------------------------------------------------- diff --git a/arrows/klv/klv_data_format.h b/arrows/klv/klv_data_format.h index e58dedde35..419fd11727 100644 --- a/arrows/klv/klv_data_format.h +++ b/arrows/klv/klv_data_format.h @@ -86,9 +86,18 @@ class KWIVER_ALGO_KLV_EXPORT klv_data_format std::string description() const; - /// Optionally the checksum format for this data format. + /// Return the checksum format for the packet key and length only, or + /// `nullptr`. virtual klv_checksum_packet_format const* - checksum_format() const; + prefix_checksum_format() const; + + /// Return the checksum format for the packet payload only, or `nullptr`. + virtual klv_checksum_packet_format const* + payload_checksum_format() const; + + /// Return the checksum format for the entire packet, or `nullptr`. + virtual klv_checksum_packet_format const* + packet_checksum_format() const; /// Return the constraints on the length of this format. klv_length_constraints const& diff --git a/arrows/klv/klv_lengthy.cxx b/arrows/klv/klv_lengthy.cxx index c7bac10cd5..d637924baf 100644 --- a/arrows/klv/klv_lengthy.cxx +++ b/arrows/klv/klv_lengthy.cxx @@ -58,52 +58,7 @@ operator<<( std::ostream& os, klv_lengthy< T > const& value ) } // ---------------------------------------------------------------------------- -template < class T > -bool -operator<( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ) -{ - return lhs.value < rhs.value; -} - -// ---------------------------------------------------------------------------- -template < class T > -bool -operator>( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ) -{ - return lhs.value > rhs.value; -} - -// ---------------------------------------------------------------------------- -template < class T > -bool -operator<=( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ) -{ - return lhs.value <= rhs.value; -} - -// ---------------------------------------------------------------------------- -template < class T > -bool -operator>=( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ) -{ - return lhs.value >= rhs.value; -} - -// ---------------------------------------------------------------------------- -template < class T > -bool -operator==( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ) -{ - return lhs.value == rhs.value; -} - -// ---------------------------------------------------------------------------- -template < class T > -bool -operator!=( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ) -{ - return lhs.value != rhs.value; -} +DEFINE_TEMPLATE_CMP( klv_lengthy< T >, &klv_lengthy< T >::value ) // ---------------------------------------------------------------------------- #define KLV_INSTANTIATE( T ) \ diff --git a/arrows/klv/klv_lengthy.h b/arrows/klv/klv_lengthy.h index ac64a333a0..39c66b3409 100644 --- a/arrows/klv/klv_lengthy.h +++ b/arrows/klv/klv_lengthy.h @@ -43,40 +43,7 @@ std::ostream& operator<<( std::ostream& os, klv_lengthy< T > const& value ); // ---------------------------------------------------------------------------- -template < class T > -KWIVER_ALGO_KLV_EXPORT -bool -operator<( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ); - -// ---------------------------------------------------------------------------- -template < class T > -KWIVER_ALGO_KLV_EXPORT -bool -operator>( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ); - -// ---------------------------------------------------------------------------- -template < class T > -KWIVER_ALGO_KLV_EXPORT -bool -operator<=( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ); - -// ---------------------------------------------------------------------------- -template < class T > -KWIVER_ALGO_KLV_EXPORT -bool -operator>=( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ); - -// ---------------------------------------------------------------------------- -template < class T > -KWIVER_ALGO_KLV_EXPORT -bool -operator==( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ); - -// ---------------------------------------------------------------------------- -template < class T > -KWIVER_ALGO_KLV_EXPORT -bool -operator!=( klv_lengthy< T > const& lhs, klv_lengthy< T > const& rhs ); +DECLARE_TEMPLATE_CMP( klv_lengthy< T > ) } // namespace klv diff --git a/arrows/klv/klv_packet.cxx b/arrows/klv/klv_packet.cxx index 3317dc7487..cf48b66246 100644 --- a/arrows/klv/klv_packet.cxx +++ b/arrows/klv/klv_packet.cxx @@ -80,87 +80,189 @@ operator<<( std::ostream& os, klv_packet const& packet ) namespace { // ---------------------------------------------------------------------------- -size_t +using data_range_t = vital::range::iterator_range< klv_read_iter_t >; + +// ---------------------------------------------------------------------------- +data_range_t verify_checksum( klv_tag_traits const& traits, - iterator_tracker< klv_read_iter_t > const& tracker, - size_t value_size ) + klv_checksum_packet_format const& format, + data_range_t data_range, + std::vector< data_range_t > wrong_ranges ) { - auto const format = traits.format().checksum_format(); - if( !format ) + auto const logger = kv::get_logger( "klv" ); + + // Get the checksum written at the end of the range + auto const checksum_size = *format.length_constraints().fixed(); + if( data_range.size() < 0 || + checksum_size > static_cast< size_t >( data_range.size() ) ) + { + LOG_ERROR( + logger, + traits.name() << ": checksum not present (data range too small)" ); + return data_range; + } + uint64_t written_checksum = 0; + try { - // This format doesn't have a checksum - return 0; + auto it = data_range.end() - checksum_size; + written_checksum = format.read_( it, checksum_size ); + } + catch( vital::metadata_exception const& e ) + { + LOG_ERROR( + logger, + traits.name() << ": could not read checksum: " << e.what() ); + return data_range; } - // Get the checksum written at the end of the packet - auto const packet_size = tracker.traversed() + value_size; - auto const checksum_size = *format->length_constraints().fixed(); - auto it = tracker.begin() + packet_size - checksum_size; - auto const written_checksum = format->read_( it, checksum_size ); - - // Calculate our own checksum over the packet - using range_t = vital::range::iterator_range< klv_read_iter_t >; - auto const header_size = format->header().size(); - range_t const checked_range{ - tracker.begin(), - tracker.begin() + packet_size - checksum_size + header_size }; + // Calculate our own checksum over the range + auto const header_size = format.header().size(); + auto const checked_size = data_range.size() - checksum_size + header_size; auto const actual_checksum = - format->evaluate( checked_range.begin(), checked_range.size() ); + format.evaluate( data_range.begin(), checked_size ); // Check for match + data_range_t const result{ + data_range.begin(), data_range.end() - checksum_size }; if( written_checksum == actual_checksum ) { - return checksum_size; + return result; } // Then check that the mismatch isn't the result of computing the checksum // over the wrong range of data. If so, this doesn't merit a full ERROR log. - for( auto const alt_begin_offset : - { int64_t{ 0 }, static_cast< int64_t >( packet_size - value_size ) } ) + for( auto const& wrong_range : wrong_ranges ) { - for( auto const alt_end_offset : - { int64_t{ 0 }, -static_cast< int64_t >( header_size ) } ) - { - if( !alt_begin_offset && !alt_end_offset ) - { - // Correct algorithm - continue; - } + auto const wrong_checksum = + format.evaluate( wrong_range.begin(), wrong_range.size() ); - // Compute the checksum of the alternate data range - range_t const alt_range{ - checked_range.begin() + alt_begin_offset, - checked_range.end() + alt_end_offset }; - auto const alt_checksum = - format->evaluate( alt_range.begin(), alt_range.size() ); - - // Check if they implemented it wrong - if( written_checksum == alt_checksum ) - { - LOG_DEBUG( - kv::get_logger( "klv" ), - traits.name() << ": " - << "the producer of this data implemented the checksum incorrectly" - ); - return checksum_size; - } + if( written_checksum == wrong_checksum ) + { + LOG_DEBUG( + logger, + traits.name() << ": " + << "the producer of this data implemented the checksum incorrectly" + ); + return result; } } // Checksum is incorrect for some unknown reason. // Possibly actual packet corruption or a different misimplementation LOG_ERROR( - kv::get_logger( "klv" ), + logger, traits.name() << ": " << "calculated checksum " - << "(" << format->to_string( actual_checksum ) << ") " + << "(" << format.to_string( actual_checksum ) << ") " << "does not equal checksum contained in packet " - << "(" << format->to_string( written_checksum ) << ")" ); + << "(" << format.to_string( written_checksum ) << ")" ); - return checksum_size; + return result; } +// ---------------------------------------------------------------------------- +data_range_t +verify_checksum( + klv_tag_traits const& traits, + iterator_tracker< klv_read_iter_t > const& tracker, + size_t value_size ) +{ + auto const key_it = tracker.begin(); + auto value_it = tracker.it(); + auto end_it = value_it + value_size; + + // Prefix first, to ensure we have the right value_size + if( auto const format = traits.format().prefix_checksum_format() ) + { + auto const checksum_size = *format->length_constraints().fixed(); + data_range_t data_range{ key_it, value_it + checksum_size }; + auto const result_range = + verify_checksum( traits, *format, data_range, { { key_it, value_it } } ); + if( result_range.end() == data_range.end() ) + { + // Failure. Since the value size could be wrong, we have to abort here. + return { value_it, value_it }; + } + value_it = data_range.end(); + } + + // Only after we check that the prefix is intact should we actually try to + // read the next value_size bytes + tracker.verify( value_size ); + + // Then entire packet + if( auto const format = traits.format().packet_checksum_format() ) + { + auto const checksum_size = *format->length_constraints().fixed(); + auto const header_size = format->header().size(); + auto const right_end_it = end_it - ( checksum_size - header_size ); + auto const wrong_end_it = end_it - checksum_size; + data_range_t data_range{ key_it, end_it }; + auto const result_range = + verify_checksum( + traits, *format, data_range, + { { key_it, wrong_end_it }, + { value_it, right_end_it }, + { value_it, wrong_end_it } } ); + end_it = result_range.end(); + } + + // Then payload (value) + if( auto const format = traits.format().payload_checksum_format() ) + { + auto const checksum_size = *format->length_constraints().fixed(); + data_range_t data_range{ value_it, end_it }; + auto const result_range = + verify_checksum( + traits, *format, data_range, + { { value_it, end_it - checksum_size } } ); + end_it = result_range.end(); + } + + // Return value range with checksums removed + return { value_it, end_it }; +} + +// ---------------------------------------------------------------------------- +size_t +all_checksums_length( klv_data_format const& format ) +{ + size_t result = 0; + for( auto const checksum_format : + { format.prefix_checksum_format(), + format.payload_checksum_format(), + format.packet_checksum_format() } ) + { + if( checksum_format ) + { + result += *checksum_format->length_constraints().fixed(); + } + } + + return result; +} + +// ---------------------------------------------------------------------------- +void +write_checksum( + klv_checksum_packet_format const* format, + klv_write_iter_t begin, klv_write_iter_t& data ) +{ + if( !format ) + { + return; + } + + auto const header = format->header(); + std::copy( header.begin(), header.end(), data ); + auto const checksum = + format->evaluate( + begin, static_cast< size_t >( data - begin ) + header.size() ); + format->write_( checksum, data, *format->length_constraints().fixed() ); +} + + } // namespace // ---------------------------------------------------------------------------- @@ -195,26 +297,27 @@ klv_read_packet( klv_read_iter_t& data, size_t max_length ) { // This might be an encoding error, or maybe we falsely detected a prefix // in the data between the packets - VITAL_THROW( kwiver::vital::metadata_exception, "invalid universal key" ); + VITAL_THROW( kv::metadata_exception, "invalid universal key" ); } // Read length auto const length_of_value = klv_read_ber< size_t >( data, tracker.remaining() ); - tracker.verify( length_of_value ); + auto const value_begin = data; // Verify checksum auto const& traits = klv_lookup_packet_traits().by_uds_key( key ); auto const& format = traits.format(); - auto const checksum_length = + auto const value_range = verify_checksum( traits, tracker, length_of_value ); // Read value + data = value_range.begin(); auto const value = - format.read( data, tracker.verify( length_of_value - checksum_length ) ); + format.read( data, tracker.verify( value_range.size() ) ); // Ensure iterator ends correctly - data += checksum_length; + data = value_begin + length_of_value; return { key, value }; } @@ -228,32 +331,23 @@ klv_write_packet( klv_packet const& packet, klv_write_iter_t& data, auto const& format = klv_lookup_packet_traits().by_uds_key( packet.key ).format(); - auto const checksum_format = format.checksum_format(); - auto const length = format.length_of( packet.value ); + auto const value_length = format.length_of( packet.value ); + auto const checksums_length = all_checksums_length( format ); auto const packet_length = klv_packet_length( packet ); - auto const checksum_length = - checksum_format ? *checksum_format->length_constraints().fixed() : 0; - if( max_length < length + checksum_length ) - { - VITAL_THROW( kwiver::vital::metadata_buffer_overflow, - "writing klv packet overflows buffer" ); - } + tracker.verify( packet_length ); + // Write prefix klv_write_uds_key( packet.key, data, tracker.remaining() ); - klv_write_ber( length + checksum_length, data, tracker.remaining() ); - format.write( packet.value, data, length ); + klv_write_ber( value_length + checksums_length, data, tracker.remaining() ); - if( checksum_format ) - { - tracker.verify( checksum_length ); - auto const header = checksum_format->header(); - std::copy( header.begin(), header.end(), data ); - auto const checksum = - checksum_format->evaluate( - tracker.begin(), - packet_length - checksum_length + header.size() ); - checksum_format->write_( checksum, data, checksum_length ); - } + write_checksum( format.prefix_checksum_format(), tracker.begin(), data ); + auto const value_begin = data; + + // Write value + format.write( packet.value, data, value_length ); + + write_checksum( format.payload_checksum_format(), value_begin, data ); + write_checksum( format.packet_checksum_format(), tracker.begin(), data ); } // ---------------------------------------------------------------------------- @@ -262,15 +356,15 @@ klv_packet_length( klv_packet const& packet ) { auto const& format = klv_lookup_packet_traits().by_uds_key( packet.key ).format(); - auto const checksum_format = format.checksum_format(); + auto const length_of_key = packet.key.length; auto const length_of_value = format.length_of( packet.value ); - auto const length_of_checksum = - checksum_format ? *checksum_format->length_constraints().fixed() : 0; + auto const length_of_checksums = all_checksums_length( format ); auto const length_of_length = - klv_ber_length( length_of_value + length_of_checksum ); + klv_ber_length( length_of_value + length_of_checksums ); + return length_of_key + length_of_length + length_of_value + - length_of_checksum; + length_of_checksums; } // ---------------------------------------------------------------------------- diff --git a/arrows/klv/klv_read_write.cxx b/arrows/klv/klv_read_write.cxx index 85c4dead65..6c0bc48c14 100644 --- a/arrows/klv/klv_read_write.cxx +++ b/arrows/klv/klv_read_write.cxx @@ -265,7 +265,14 @@ klv_read_imap( // Return exactly zero if applicable, overriding rounding errors. IMAP // specification considers this important auto const precision = klv_imap_precision( interval, length ); - return ( std::abs( value ) < precision / 2.0 ) ? 0.0 : value; + value = ( std::abs( value ) < precision / 2.0 ) ? 0.0 : value; + + if( !interval.contains( value, true, true ) ) + { + VITAL_THROW( kv::metadata_type_overflow, "value outside IMAP bounds" ); + } + + return value; } // ---------------------------------------------------------------------------- diff --git a/arrows/klv/klv_read_write.h b/arrows/klv/klv_read_write.h index 381fa98c8c..546fd4c88d 100644 --- a/arrows/klv/klv_read_write.h +++ b/arrows/klv/klv_read_write.h @@ -335,7 +335,8 @@ klv_write_float( double value, klv_write_iter_t& data, size_t length ); /// \returns Floating-point number decoded from \p data buffer. /// /// \throws metadata_type_overflow When \p length is greater than the size of a -/// \c uint64_t or the span of \p interval is too large for a \c double to hold. +/// \c uint64_t, or the span of \p interval is too large for a \c double to +/// hold, or the result value would fall outside \p interval. KWIVER_ALGO_KLV_EXPORT double klv_read_imap( diff --git a/arrows/klv/klv_set.cxx b/arrows/klv/klv_set.cxx index 5d927e6e8b..cfc1e0c8ec 100644 --- a/arrows/klv/klv_set.cxx +++ b/arrows/klv/klv_set.cxx @@ -373,6 +373,15 @@ klv_set_format< Key > ::~klv_set_format() {} +// ---------------------------------------------------------------------------- +template < class Key > +klv_tag_traits_lookup const& +klv_set_format< Key > +::traits() const +{ + return m_traits; +} + // ---------------------------------------------------------------------------- template < class Key > klv_set< Key > diff --git a/arrows/klv/klv_set.h b/arrows/klv/klv_set.h index e847e62aed..79c8705270 100644 --- a/arrows/klv/klv_set.h +++ b/arrows/klv/klv_set.h @@ -164,7 +164,9 @@ class KWIVER_ALGO_KLV_EXPORT klv_set_format virtual ~klv_set_format(); -protected: + klv_tag_traits_lookup const& + traits() const; + klv_set< Key > read_typed( klv_read_iter_t& data, size_t length ) const override; @@ -178,6 +180,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_set_format std::ostream& print_typed( std::ostream& os, klv_set< Key > const& value ) const; +protected: // Print warnings if tags appear too few or too many times in the given set. void check_tag_counts( klv_set< Key > const& klv ) const; @@ -185,6 +188,7 @@ class KWIVER_ALGO_KLV_EXPORT klv_set_format virtual void check_set( klv_set< Key > const& klv ) const; +private: klv_tag_traits_lookup const& m_traits; }; diff --git a/arrows/klv/klv_stream_settings.cxx b/arrows/klv/klv_stream_settings.cxx new file mode 100644 index 0000000000..4c26b21ef3 --- /dev/null +++ b/arrows/klv/klv_stream_settings.cxx @@ -0,0 +1,36 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Implementation of settings structure for the creation of a klv stream. + +#include "klv_stream_settings.h" + +#include + +namespace kwiver { + +namespace arrows { + +namespace klv { + +// ---------------------------------------------------------------------------- +klv_stream_settings +::klv_stream_settings() + : type{ KLV_STREAM_TYPE_ASYNC }, + index{ INT_MIN } +{} + +// ---------------------------------------------------------------------------- +bool operator==( klv_stream_settings const& lhs, + klv_stream_settings const& rhs ) +{ + return lhs.type == rhs.type && lhs.index == rhs.index; +} + +} // namespace klv + +} // namespace arrows + +} // namespace kwiver diff --git a/arrows/klv/klv_stream_settings.h b/arrows/klv/klv_stream_settings.h new file mode 100644 index 0000000000..41349160ad --- /dev/null +++ b/arrows/klv/klv_stream_settings.h @@ -0,0 +1,52 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Declaration of settings structure for the creation of a klv stream. + +#ifndef KWIVER_ARROWS_KLV_KLV_STREAM_SETTINGS_H_ +#define KWIVER_ARROWS_KLV_KLV_STREAM_SETTINGS_H_ + +#include + +namespace kwiver { + +namespace arrows { + +namespace klv { + +// ---------------------------------------------------------------------------- +/// Synchronicity of a KLV stream. +enum klv_stream_type { + KLV_STREAM_TYPE_SYNC, + KLV_STREAM_TYPE_ASYNC +}; + +// ---------------------------------------------------------------------------- +/// Parameters describing the general characteristics of a KLV stream. +/// +/// Members have been left public so users may modify them at their disgression. +struct KWIVER_ALGO_KLV_EXPORT klv_stream_settings { + klv_stream_settings(); + + /// Whether this stream is synchronous or asynchronous. + klv_stream_type type; + + /// Index of this stream in the input file. May not determine the index in + /// an output file. + int index; +}; + +// ---------------------------------------------------------------------------- +KWIVER_ALGO_KLV_EXPORT +bool operator==( klv_stream_settings const& lhs, + klv_stream_settings const& rhs ); + +} // namespace klv + +} // namespace arrows + +} // namespace kwiver + +#endif diff --git a/arrows/klv/klv_tag_traits.cxx b/arrows/klv/klv_tag_traits.cxx index 3a803c5608..55ece84d80 100644 --- a/arrows/klv/klv_tag_traits.cxx +++ b/arrows/klv/klv_tag_traits.cxx @@ -139,6 +139,12 @@ klv_tag_traits_lookup const* klv_tag_traits ::subtag_lookup() const { return m_subtag_lookup; } +// ---------------------------------------------------------------------------- +klv_tag_traits_lookup +::klv_tag_traits_lookup() + : m_traits{} +{} + // ---------------------------------------------------------------------------- klv_tag_traits_lookup ::klv_tag_traits_lookup( std::initializer_list< klv_tag_traits > const& traits ) @@ -155,6 +161,28 @@ ::klv_tag_traits_lookup( std::vector< klv_tag_traits > const& traits ) initialize(); } +// ---------------------------------------------------------------------------- +klv_tag_traits_lookup +::klv_tag_traits_lookup( klv_tag_traits_lookup const& other ) + : m_traits( other.m_traits ) +{ + initialize(); +} + +// ---------------------------------------------------------------------------- +klv_tag_traits_lookup& +klv_tag_traits_lookup +::operator=( klv_tag_traits_lookup const& other ) +{ + m_tag_to_traits.clear(); + m_uds_key_to_traits.clear(); + m_name_to_traits.clear(); + m_enum_name_to_traits.clear(); + m_traits = other.m_traits; + initialize(); + return *this; +} + // ---------------------------------------------------------------------------- typename klv_tag_traits_lookup::iterator klv_tag_traits_lookup @@ -220,11 +248,6 @@ void klv_tag_traits_lookup ::initialize() { - if( m_traits.empty() ) - { - throw std::logic_error( "tag traits cannot be empty" ); - } - for( auto const& trait : m_traits ) { if( trait.tag() && !m_tag_to_traits.emplace( trait.tag(), &trait ).second ) diff --git a/arrows/klv/klv_tag_traits.h b/arrows/klv/klv_tag_traits.h index 4408ba5ac2..bd7b0e1ebe 100644 --- a/arrows/klv/klv_tag_traits.h +++ b/arrows/klv/klv_tag_traits.h @@ -127,6 +127,8 @@ class KWIVER_ALGO_KLV_EXPORT klv_tag_traits_lookup public: using iterator = typename std::vector< klv_tag_traits >::const_iterator; + klv_tag_traits_lookup(); + /// Create lookup tables for the tag, uds_key, name, and enum name of /// \p traits. /// @@ -145,6 +147,11 @@ class KWIVER_ALGO_KLV_EXPORT klv_tag_traits_lookup /// klv_tag_traits_lookup( std::initializer_list< klv_tag_traits > const& ) klv_tag_traits_lookup( std::vector< klv_tag_traits > const& traits ); + klv_tag_traits_lookup( klv_tag_traits_lookup const& other ); + + klv_tag_traits_lookup& + operator=( klv_tag_traits_lookup const& other ); + iterator begin() const; diff --git a/arrows/klv/klv_util.h b/arrows/klv/klv_util.h index 72e41e9b3a..71d7e60314 100644 --- a/arrows/klv/klv_util.h +++ b/arrows/klv/klv_util.h @@ -85,12 +85,68 @@ operator<<( std::ostream& os, std::set< T > const& value ) return os; } +// ---------------------------------------------------------------------------- +template< class T > +struct wrap_cmp_nan +{ + explicit wrap_cmp_nan( T const& value ) : value{ value } {} + + bool operator<( wrap_cmp_nan< T > const& other ) const + { + if constexpr( std::is_floating_point_v< T > ) + { + return value < other.value || + ( std::isnan( value ) && !std::isnan( other.value ) ); + } + else + { + return value < other.value; + } + } + + bool operator>( wrap_cmp_nan< T > const& other ) const + { + return other < *this; + } + + bool operator<=( wrap_cmp_nan< T > const& other ) const + { + return !( other < *this ); + } + + bool operator>=( wrap_cmp_nan< T > const& other ) const + { + return !( *this < other ); + } + + bool operator==( wrap_cmp_nan< T > const& other ) const + { + if constexpr( std::is_floating_point_v< T > ) + { + return value == other.value || + ( std::isnan( value ) && std::isnan( other.value ) ); + } + else + { + return value == other.value; + } + } + + bool operator!=( wrap_cmp_nan< T > const& other ) const + { + return !( *this == other ); + } + + T const& value; +}; + // ---------------------------------------------------------------------------- template< class T, class... Args > bool struct_lt( T const& lhs, T const& rhs, Args T::*... args ) { - return std::tie( ( lhs.*args )... ) < std::tie( ( rhs.*args )... ); + return std::make_tuple( wrap_cmp_nan( lhs.*args )... ) < + std::make_tuple( wrap_cmp_nan( rhs.*args )... ); } // ---------------------------------------------------------------------------- @@ -98,7 +154,8 @@ template< class T, class... Args > bool struct_gt( T const& lhs, T const& rhs, Args T::*... args ) { - return std::tie( ( lhs.*args )... ) > std::tie( ( rhs.*args )... ); + return std::make_tuple( wrap_cmp_nan( lhs.*args )... ) > + std::make_tuple( wrap_cmp_nan( rhs.*args )... ); } // ---------------------------------------------------------------------------- @@ -106,7 +163,8 @@ template< class T, class... Args > bool struct_le( T const& lhs, T const& rhs, Args T::*... args ) { - return std::tie( ( lhs.*args )... ) <= std::tie( ( rhs.*args )... ); + return std::make_tuple( wrap_cmp_nan( lhs.*args )... ) <= + std::make_tuple( wrap_cmp_nan( rhs.*args )... ); } // ---------------------------------------------------------------------------- @@ -114,7 +172,8 @@ template< class T, class... Args > bool struct_ge( T const& lhs, T const& rhs, Args T::*... args ) { - return std::tie( ( lhs.*args )... ) >= std::tie( ( rhs.*args )... ); + return std::make_tuple( wrap_cmp_nan( lhs.*args )... ) >= + std::make_tuple( wrap_cmp_nan( rhs.*args )... ); } // ---------------------------------------------------------------------------- @@ -122,7 +181,8 @@ template< class T, class... Args > bool struct_eq( T const& lhs, T const& rhs, Args T::*... args ) { - return std::tie( ( lhs.*args )... ) == std::tie( ( rhs.*args )... ); + return std::make_tuple( wrap_cmp_nan( lhs.*args )... ) == + std::make_tuple( wrap_cmp_nan( rhs.*args )... ); } // ---------------------------------------------------------------------------- @@ -130,7 +190,8 @@ template< class T, class... Args > bool struct_ne( T const& lhs, T const& rhs, Args T::*... args ) { - return std::tie( ( lhs.*args )... ) != std::tie( ( rhs.*args )... ); + return std::make_tuple( wrap_cmp_nan( lhs.*args )... ) != + std::make_tuple( wrap_cmp_nan( rhs.*args )... ); } // ---------------------------------------------------------------------------- @@ -186,27 +247,54 @@ class iterator_tracker { : m_begin( it ), m_length{ length }, m_it( it ) { static_assert( - std::is_same< typename std::decay< decltype( *it ) >::type, - uint8_t >::value, "iterator must point to uint8_t" ); + std::is_same_v< std::decay_t< decltype( *it ) >, uint8_t >, + "iterator must point to uint8_t" ); } - size_t verify( size_t count ) const + template< + class Int, + std::enable_if_t< std::is_unsigned_v< std::decay_t< Int > >, bool > = true > + size_t verify( Int count ) const { if( count > remaining() ) { m_it = m_begin; - VITAL_THROW( kwiver::vital::metadata_buffer_overflow, + VITAL_THROW( vital::metadata_buffer_overflow, "tried to read or write past end of data buffer" ); } return count; } + template< + class Int, + std::enable_if_t< std::is_signed_v< std::decay_t< Int > >, bool > = true > + size_t verify( Int count ) const + { + if( count < 0 ) + { + m_it = m_begin; + VITAL_THROW( vital::metadata_buffer_overflow, + "tried to read or write a value of negative length" ); + } + + return verify( static_cast< size_t >( count ) ); + } + size_t traversed() const { auto const distance = std::distance( m_begin, m_it ); - if( distance > m_length ) + + if( distance < 0 ) { - VITAL_THROW( kwiver::vital::metadata_buffer_overflow, + m_it = m_begin; + VITAL_THROW( vital::metadata_buffer_overflow, + "read or written before beginning of data buffer" ); + } + + if( static_cast< size_t >( distance ) > m_length ) + { + m_it = m_begin; + VITAL_THROW( vital::metadata_buffer_overflow, "read or written past end of data buffer" ); } diff --git a/arrows/klv/klv_value.cxx b/arrows/klv/klv_value.cxx index 64976d9d32..14fdef2e52 100644 --- a/arrows/klv/klv_value.cxx +++ b/arrows/klv/klv_value.cxx @@ -12,6 +12,7 @@ #include "klv_packet.h" #include "klv_series.hpp" #include "klv_set.h" +#include "klv_util.h" #include @@ -21,30 +22,6 @@ namespace arrows { namespace klv { -namespace { - -// ---------------------------------------------------------------------------- -template < class T, - typename std::enable_if< !std::is_floating_point< T >::value, - bool >::type = true > -bool -equal_or_nan( T const& lhs, T const& rhs ) -{ - return lhs == rhs; -} - -// ---------------------------------------------------------------------------- -template < class T, - typename std::enable_if< std::is_floating_point< T >::value, - bool >::type = true > -bool -equal_or_nan( T const& lhs, T const& rhs ) -{ - return lhs == rhs || ( std::isnan( lhs ) && std::isnan( rhs ) ); -} - -} // namespace - // ---------------------------------------------------------------------------- klv_bad_value_cast ::klv_bad_value_cast( std::type_info const& requested_type, @@ -130,7 +107,7 @@ class klv_value::internal_ : public internal_base auto const& rhs_item = dynamic_cast< internal_< T > const& >( rhs ).m_item; // Second, compare values - return equal_or_nan( lhs.m_item, rhs_item ); + return wrap_cmp_nan( lhs.m_item ) == wrap_cmp_nan( rhs_item ); } std::ostream& diff --git a/doc/manuals/rtd_requirement.txt b/doc/manuals/rtd_requirement.txt index cd6467ed82..4ed25c6e03 100644 --- a/doc/manuals/rtd_requirement.txt +++ b/doc/manuals/rtd_requirement.txt @@ -1 +1,3 @@ breathe +sphinx<7.0.0 +sphinx_rtd_theme diff --git a/doc/release-notes/master.txt b/doc/release-notes/master.txt index af7ea9690d..4897606d3e 100644 --- a/doc/release-notes/master.txt +++ b/doc/release-notes/master.txt @@ -6,3 +6,63 @@ over the previous v1.8.0 release. Updates ------- + +Vital: Types + +* Added interfaces for copying uninterpreted video data. + +Arrows: Core + +* Implemented the csv_reader reading std::optional fields. + +* Increased the precision of written floats and doubles in the csv_writer. + +* Made the transcode applet's failure to open a video result in a more graceful exit. + +* Made the transcode applet ask for video settings slightly later, when they + might be more accurate. + +* Added pass-throughs for uninterpreted video data in the metadata_filter and + buffered_metadata_filter video inputs. + +Arrows: FFmpeg + +* Added check for incoming raw video packets' timestamps and stream indices. + +* Added functionality to copy the input video's start timestamp when transcoding. + +* Removed imagery_enabled option from ffmpeg_video_input. + +* Added proper handling for changing size or pixel format mid-video. + +* Improved probing behavior and increased duration. + +* Ensured proper clearing of video state even when seek attempts fail. + +* Exposed codec options in ffmpeg_video_settings. + +* Optimized copying behavior when encoding frames. + +* Implemented packet lookahead to ensure stream synchronization. + +* Implemented copy of audio streams from input to output. + +* Fixed bugs causing invalid output when transcoding between .mp4 and .ts files. + +Arrows: KLV + +* Ensured that NaN comparisons happen consistently across all data structures. + +* Throw an exception when reading an out-of-bounds IMAP value. + +* Made the compare-klv applet's failure to open a video result in a more graceful exit. + +* Disabled initialization of image writer in dump-klv if no images are to be written. + +* Accounted for missing or extra frames in compare-klv. + +* Implemented metadata_map_io_csv.load_(). + +* Modified flint, IMAP, and integer behavior to print a warning when writing + values with incorrect lengths instead of correcting the length and possibly + losing data. diff --git a/examples/cpp/how_to_part_01_images.cpp b/examples/cpp/how_to_part_01_images.cpp index 1ea62f3b92..d669c76d83 100644 --- a/examples/cpp/how_to_part_01_images.cpp +++ b/examples/cpp/how_to_part_01_images.cpp @@ -72,19 +72,17 @@ void how_to_part_01_images() // And that we tell our application CMake targets about OpenCV (See the CMakeLists.txt for this file) cv::Mat mat; // First, convert the image to an OpenCV image object - mat = kwiver::arrows::ocv::image_container::vital_to_ocv(ocv_img->get_image(), kwiver::arrows::ocv::image_container::RGB_COLOR ); + mat = kwiver::arrows::ocv::image_container::vital_to_ocv(ocv_img->get_image(), kwiver::arrows::ocv::image_container::BGR_COLOR ); cv::namedWindow("Image loaded by OpenCV", cv::WINDOW_AUTOSIZE);// Create a window for display. cv::imshow("Image loaded by OpenCV", mat); // Show our image inside it. - cv::waitKey(5); - kwiversys::SystemTools::Delay(2000); // Wait for 2s + cv::waitKey(2000); // Wait for 2s cv::destroyWindow("Image loaded by OpenCV"); // We can do the same, even if the image was originally loaded with VXL - mat = kwiver::arrows::ocv::image_container::vital_to_ocv(vxl_img->get_image(), kwiver::arrows::ocv::image_container::RGB_COLOR); + mat = kwiver::arrows::ocv::image_container::vital_to_ocv(vxl_img->get_image(), kwiver::arrows::ocv::image_container::BGR_COLOR); cv::namedWindow("Image loaded by VXL", cv::WINDOW_AUTOSIZE);// Create a window for display. cv::imshow("Image loaded by VXL", mat); // Show our image inside it. - cv::waitKey(5); - kwiversys::SystemTools::Delay(2000); // Wait for 2s + cv::waitKey(2000); // Wait for 2s cv::destroyWindow("Image loaded by VXL"); ////////////////// @@ -105,22 +103,20 @@ void how_to_part_01_images() std::vector ocv_imgs = ocv_split->split(vxl_img); for (kwiver::vital::image_container_sptr i : ocv_imgs) { - mat = kwiver::arrows::ocv::image_container::vital_to_ocv(i->get_image(), kwiver::arrows::ocv::image_container::RGB_COLOR); + mat = kwiver::arrows::ocv::image_container::vital_to_ocv(i->get_image(), kwiver::arrows::ocv::image_container::BGR_COLOR); cv::namedWindow("OpenCV Split Image", cv::WINDOW_AUTOSIZE);// Create a window for display. cv::imshow("OpenCV Split Image", mat); // Show our image inside it. - cv::waitKey(5); - kwiversys::SystemTools::Delay(2000); // Wait for 2s + cv::waitKey(2000); // Wait for 2s cv::destroyWindow("OpenCV Split Image"); } std::vector vxl_imgs = ocv_split->split(ocv_img); for (kwiver::vital::image_container_sptr i : vxl_imgs) { - mat = kwiver::arrows::ocv::image_container::vital_to_ocv(i->get_image(), kwiver::arrows::ocv::image_container::RGB_COLOR); + mat = kwiver::arrows::ocv::image_container::vital_to_ocv(i->get_image(), kwiver::arrows::ocv::image_container::BGR_COLOR); cv::namedWindow("VXL Split Image", cv::WINDOW_AUTOSIZE);// Create a window for display. cv::imshow("VXL Split Image", mat); // Show our image inside it. - cv::waitKey(5); - kwiversys::SystemTools::Delay(2000); // Wait for 2s + cv::waitKey(2000); // Wait for 2s cv::destroyWindow("VXL Split Image"); } diff --git a/examples/cpp/how_to_part_02_detections.cpp b/examples/cpp/how_to_part_02_detections.cpp index 29dce33426..f4171dadba 100644 --- a/examples/cpp/how_to_part_02_detections.cpp +++ b/examples/cpp/how_to_part_02_detections.cpp @@ -50,8 +50,7 @@ void how_to_part_02_detections() cv::Mat hough_mat = kwiver::arrows::ocv::image_container::vital_to_ocv(hough_img->get_image(), kwiver::arrows::ocv::image_container::RGB_COLOR); cv::namedWindow("Hough Detections", cv::WINDOW_AUTOSIZE);// Create a window for display. cv::imshow("Hough Detections", hough_mat); // Show our image inside it. - cv::waitKey(5); - kwiversys::SystemTools::Delay(2000); // Wait for 2s + cv::waitKey(2000); // Wait for 2s cv::destroyWindow("Hough Detections"); // Next, let's look at the detection data structures and we can make them @@ -116,8 +115,7 @@ void how_to_part_02_detections() cv::Mat mat = kwiver::arrows::ocv::image_container::vital_to_ocv(img_detections->get_image(), kwiver::arrows::ocv::image_container::RGB_COLOR); cv::namedWindow("Detections", cv::WINDOW_AUTOSIZE);// Create a window for display. cv::imshow("Detections", mat); // Show our image inside it. - cv::waitKey(5); - kwiversys::SystemTools::Delay(2000); // Wait for 2s + cv::waitKey(2000); // Wait for 2s cv::destroyWindow("Detections"); kwiver::vital::algo::detected_object_set_output_sptr kpf_writer = kwiver::vital::algo::detected_object_set_output::create("kpf_output"); diff --git a/python/kwiver/vital/algo/trampoline/detected_object_set_input_trampoline.txx b/python/kwiver/vital/algo/trampoline/detected_object_set_input_trampoline.txx index c13ba56f78..0bcd9bb7a9 100644 --- a/python/kwiver/vital/algo/trampoline/detected_object_set_input_trampoline.txx +++ b/python/kwiver/vital/algo/trampoline/detected_object_set_input_trampoline.txx @@ -56,14 +56,14 @@ class detected_object_set_input_trampoline : pybind11::gil_scoped_acquire gil; pybind11::function overload = pybind11::get_overload(static_cast(this), "read_set"); if (overload) { - auto o = overload(); - if (pybind11::isinstance(o)) { - return false; - } - std::tie(set, image_path) = o.cast>(); - return true; + auto o = overload(); + if (pybind11::isinstance(o)) { + return false; + } + std::tie(set, image_path) = o.cast>(); + return true; } else { - pybind11::pybind11_fail("Tried to call pure virtual function \"dosi::read_set\""); + pybind11::pybind11_fail("Tried to call pure virtual function \"dosi::read_set\""); } } @@ -73,7 +73,7 @@ class detected_object_set_input_trampoline : void, dosi, open, - filename + filename ); } diff --git a/python/kwiver/vital/algo/trampoline/detected_object_set_output_trampoline.txx b/python/kwiver/vital/algo/trampoline/detected_object_set_output_trampoline.txx index a43a196610..4e9ccde528 100644 --- a/python/kwiver/vital/algo/trampoline/detected_object_set_output_trampoline.txx +++ b/python/kwiver/vital/algo/trampoline/detected_object_set_output_trampoline.txx @@ -71,10 +71,10 @@ class detected_object_set_output_trampoline : void complete() override { PYBIND11_OVERLOAD( - void, - kwiver::vital::algo::detected_object_set_output, - complete, - ); + void, + kwiver::vital::algo::detected_object_set_output, + complete, + ); } void diff --git a/python/kwiver/vital/util/python_exceptions.h b/python/kwiver/vital/util/python_exceptions.h index 704856816c..3def6587ba 100644 --- a/python/kwiver/vital/util/python_exceptions.h +++ b/python/kwiver/vital/util/python_exceptions.h @@ -51,7 +51,7 @@ void VITAL_PYTHON_UTIL_EXPORT python_print_exception(); } \ catch ( std::exception const& e ) \ { \ - pybind11::gil_scoped_acquire acquire; \ + pybind11::gil_scoped_acquire acquire; \ (void) acquire; \ PyErr_SetString( PyExc_RuntimeError, e.what() ); \ \ diff --git a/test_data/videos/h264_audio.ts b/test_data/videos/h264_audio.ts new file mode 100644 index 0000000000..a7d5864499 Binary files /dev/null and b/test_data/videos/h264_audio.ts differ diff --git a/vital/CMakeLists.txt b/vital/CMakeLists.txt index c8960bc77a..54646cc2ed 100644 --- a/vital/CMakeLists.txt +++ b/vital/CMakeLists.txt @@ -166,6 +166,7 @@ set( vital_public_headers types/video_raw_image.h types/video_raw_metadata.h types/video_settings.h + types/video_uninterpreted_data.h ) # ---------------------- @@ -236,6 +237,7 @@ set( vital_sources types/video_raw_image.cxx types/video_raw_metadata.cxx types/video_settings.cxx + types/video_uninterpreted_data.cxx types/uid.cxx ) diff --git a/vital/algo/video_input.cxx b/vital/algo/video_input.cxx index 8856f7889b..613302ff93 100644 --- a/vital/algo/video_input.cxx +++ b/vital/algo/video_input.cxx @@ -27,6 +27,7 @@ algorithm_capabilities::capability_name_t const video_input::HAS_TIMEOUT( "has-t algorithm_capabilities::capability_name_t const video_input::IS_SEEKABLE( "is-seekable" ); algorithm_capabilities::capability_name_t const video_input::HAS_RAW_IMAGE( "has-raw-image" ); algorithm_capabilities::capability_name_t const video_input::HAS_RAW_METADATA( "has-raw-metadata" ); +algorithm_capabilities::capability_name_t const video_input::HAS_UNINTERPRETED_DATA( "has-uninterpreted-data" ); // ---------------------------------------------------------------------------- @@ -68,6 +69,14 @@ ::raw_frame_metadata() return nullptr; } +// ---------------------------------------------------------------------------- +video_uninterpreted_data_sptr +video_input +::uninterpreted_frame_data() +{ + return nullptr; +} + // ---------------------------------------------------------------------------- video_settings_uptr video_input diff --git a/vital/algo/video_input.h b/vital/algo/video_input.h index 01079ca970..1c7a1b6ca9 100644 --- a/vital/algo/video_input.h +++ b/vital/algo/video_input.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -123,6 +124,7 @@ class VITAL_ALGO_EXPORT video_input static const algorithm_capabilities::capability_name_t IS_SEEKABLE; static const algorithm_capabilities::capability_name_t HAS_RAW_IMAGE; static const algorithm_capabilities::capability_name_t HAS_RAW_METADATA; + static const algorithm_capabilities::capability_name_t HAS_UNINTERPRETED_DATA; virtual ~video_input(); @@ -344,6 +346,16 @@ class VITAL_ALGO_EXPORT video_input /// \return Pointer to raw metadata. virtual video_raw_metadata_sptr raw_frame_metadata(); + /// Return an implementation-defined representation of uninterpreted data in + /// this frame. + /// + /// This method enables passage of miscellaneous data - such as audio, + /// unrecognized metadata, or secondary image streams - to a video output when + /// transcoding. + /// + /// \return Pointer to uninterpreted data. + virtual video_uninterpreted_data_sptr uninterpreted_frame_data(); + /// \brief Get metadata map for video. /// /// This method returns a metadata map for the video assuming the video is diff --git a/vital/algo/video_output.cxx b/vital/algo/video_output.cxx index 784386e3f5..682048c34d 100644 --- a/vital/algo/video_output.cxx +++ b/vital/algo/video_output.cxx @@ -27,6 +27,10 @@ video_output::SUPPORTS_FRAME_TIME( "supports-frame-time" ); const algorithm_capabilities::capability_name_t video_output::SUPPORTS_METADATA( "supports-metadata" ); +// ---------------------------------------------------------------------------- +const algorithm_capabilities::capability_name_t +video_output::SUPPORTS_UNINTERPRETED_DATA( "supports-uninterpreted-data" ); + // ---------------------------------------------------------------------------- video_output ::video_output() @@ -58,6 +62,15 @@ ::add_metadata( video_raw_metadata const& md ) "video_output: This implementation does not support raw metadata" }; } +// ---------------------------------------------------------------------------- +void +video_output +::add_uninterpreted_data( video_uninterpreted_data const& misc_data ) +{ + throw std::logic_error{ + "video_output: This implementation does not support uninterpreted data" }; +} + // ---------------------------------------------------------------------------- vital::video_settings_uptr video_output diff --git a/vital/algo/video_output.h b/vital/algo/video_output.h index 40126d7b94..b9571249ca 100644 --- a/vital/algo/video_output.h +++ b/vital/algo/video_output.h @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -61,6 +62,13 @@ class VITAL_ALGO_EXPORT video_output /// metadata. static const algorithm_capabilities::capability_name_t SUPPORTS_METADATA; + /// Writer can write uninterpreted data. + /// + /// This capability indicates if the implementation can take data which a + /// video input did not interpret and put it back in the video stream. + static const algorithm_capabilities::capability_name_t + SUPPORTS_UNINTERPRETED_DATA; + virtual ~video_output(); /// Return the name of this algorithm. @@ -132,11 +140,14 @@ class VITAL_ALGO_EXPORT video_output /// This method writes the raw metadata to the video stream. There is no /// guarantee that this functions correctly when intermixed with non-raw /// metadata. - /// - /// For implementations that do not support metadata, this method does - /// nothing. virtual void add_metadata( video_raw_metadata const& md ); + /// Add a frame of uninterpreted data to the video stream. + /// + /// This method writes the uninterpreted data to the video stream. + virtual void add_uninterpreted_data( + video_uninterpreted_data const& misc_data ); + /// Extract implementation-specific video encoding settings. /// /// The returned structure is intended to be passed to a video encoder of diff --git a/vital/io/mesh_io.cxx b/vital/io/mesh_io.cxx index 10dc674e77..9b3300c5e1 100644 --- a/vital/io/mesh_io.cxx +++ b/vital/io/mesh_io.cxx @@ -20,19 +20,18 @@ namespace vital { namespace { -/// Helper function to check that the output file name can be used -void -check_output_file(const std::string& filename) +/// Helper function to open the output file and return its stream +std::ofstream +open_output_file(const std::string& filename) { - // If the given path is a directory, we obviously can't write to it. + // Check if the given path is a directory if ( kwiversys::SystemTools::FileIsDirectory( filename ) ) { VITAL_THROW( file_write_exception, filename, - "Path given is a directory, can not write file." ); + "Path is a directory, cannot write file." ); } - // Check that the directory of the given filepath exists, creating necessary - // directories where needed. + // Ensure the directory of the given filepath exists, create if necessary std::string parent_dir = kwiversys::SystemTools::GetFilenamePath( kwiversys::SystemTools::CollapseFullPath( filename )); if ( ! kwiversys::SystemTools::FileIsDirectory( parent_dir ) ) @@ -40,37 +39,46 @@ check_output_file(const std::string& filename) if ( ! kwiversys::SystemTools::MakeDirectory( parent_dir ) ) { VITAL_THROW( file_write_exception, parent_dir, - "Attempted directory creation, but no directory created!" ); + "Failed to create directory." ); } } - // Open the output stream - std::ofstream output_stream(filename.c_str()); + // Open the output file stream + std::ofstream output_stream(filename.c_str(), std::fstream::out); + if (!output_stream) + { + VITAL_THROW( file_write_exception, filename, + "Could not open file for writing." ); + } + + return output_stream; } -/// Helper function to check that the input file name can be used -void -check_input_file(const std::string& filename) +/// Helper function to open the input file and return its stream +std::ifstream +open_input_file(const std::string& filename) { - // Check that file exists + // Check that file exists and is not a directory if ( ! kwiversys::SystemTools::FileExists( filename ) ) { VITAL_THROW( file_not_found_exception, filename, "File does not exist." ); } - else if ( kwiversys::SystemTools::FileIsDirectory( filename ) ) + else if ( kwiversys::SystemTools::FileIsDirectory( filename ) ) { VITAL_THROW( file_not_found_exception, filename, "Path given doesn't point to a regular file!" ); } - // Reading in input file data + // Open the input file stream std::ifstream input_stream( filename.c_str(), std::fstream::in ); if ( ! input_stream ) { VITAL_THROW( file_not_read_exception, filename, "Could not open file at given path." ); } + + return input_stream; } } @@ -79,9 +87,9 @@ check_input_file(const std::string& filename) mesh_sptr read_mesh(const std::string& filename) { - check_input_file(filename); - std::ifstream input_stream(filename.c_str()); + std::ifstream input_stream = open_input_file(filename); const std::string ext = kwiversys::SystemTools::GetFilenameLastExtension(filename); + if (ext == ".ply2") { return read_ply2(input_stream); @@ -95,15 +103,16 @@ read_mesh(const std::string& filename) return read_obj(input_stream); } - return mesh_sptr(); + VITAL_THROW( invalid_file, filename, + "Unrecognized file extension for mesh file"); } /// Read a mesh from a PLY2 file mesh_sptr read_ply2(const std::string& filename) { - check_input_file(filename); - std::ifstream input_stream(filename.c_str()); + std::ifstream input_stream = open_input_file(filename); + return read_ply2(input_stream); } @@ -139,8 +148,8 @@ read_ply2(std::istream& is) mesh_sptr read_ply(const std::string& filename) { - check_input_file(filename); - std::ifstream input_stream(filename.c_str()); + std::ifstream input_stream = open_input_file(filename); + return read_ply(input_stream); } @@ -198,8 +207,8 @@ mesh_sptr read_ply(std::istream& is) void write_ply2(const std::string& filename, const mesh& mesh) { - check_output_file(filename); - std::ofstream output_stream(filename.c_str()); + std::ofstream output_stream = open_output_file(filename); + write_ply2(output_stream, mesh); } @@ -230,10 +239,9 @@ void write_ply2(std::ostream& os, const mesh& mesh) /// Read texture coordinates from a UV2 file bool read_uv2(const std::string& filename, mesh& mesh) { - std::ifstream fh(filename.c_str()); - bool retval = read_uv2(fh,mesh); - fh.close(); - return retval; + std::ifstream input_stream = open_input_file(filename); + + return read_uv2(input_stream, mesh); } /// Read texture coordinates from a UV2 stream @@ -261,8 +269,8 @@ bool read_uv2(std::istream& is, mesh& mesh) mesh_sptr read_obj(const std::string& filename) { - check_input_file(filename); - std::ifstream input_stream(filename.c_str()); + std::ifstream input_stream = open_input_file(filename); + return read_obj(input_stream); } @@ -341,7 +349,8 @@ read_obj(std::istream& is) else { LOG_ERROR(logger, "improperly formed face line in OBJ: "< ( fp ); - ( *reg_fp )( m_parent ); // register plugins + if( m_parent ) + { + ( *reg_fp )( *m_parent ); // register plugins + } } // ---------------------------------------------------------------------------- diff --git a/vital/tests/test_mesh_io.cxx b/vital/tests/test_mesh_io.cxx index 601fa25506..9a7f83c7ee 100644 --- a/vital/tests/test_mesh_io.cxx +++ b/vital/tests/test_mesh_io.cxx @@ -56,10 +56,9 @@ TEST_F(mesh_io, invalid_input_file) // ---------------------------------------------------------------------------- TEST_F(mesh_io, read_invalid_type) { - mesh_sptr empty_read_mesh = read_mesh( - data_dir + "/aphill_pipeline_data/geo_origin.txt"); - - EXPECT_EQ( empty_read_mesh, nullptr ); + EXPECT_THROW( + read_mesh(data_dir + "/aphill_pipeline_data/geo_origin.txt"), invalid_file + ); } // ---------------------------------------------------------------------------- diff --git a/vital/types/class_map.h b/vital/types/class_map.h index 0d0101b8ac..9b6bedf322 100644 --- a/vital/types/class_map.h +++ b/vital/types/class_map.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/vital/types/color.h b/vital/types/color.h index d61b9b9cdb..04b802d210 100644 --- a/vital/types/color.h +++ b/vital/types/color.h @@ -10,6 +10,8 @@ #include +#include + namespace kwiver { namespace vital { diff --git a/vital/types/video_uninterpreted_data.cxx b/vital/types/video_uninterpreted_data.cxx new file mode 100644 index 0000000000..5164de9644 --- /dev/null +++ b/vital/types/video_uninterpreted_data.cxx @@ -0,0 +1,21 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Implementation of base video uninterpreted data type. + +#include + +namespace kwiver { + +namespace vital { + +// ---------------------------------------------------------------------------- +video_uninterpreted_data +::~video_uninterpreted_data() +{} + +} // namespace vital + +} // namespace kwiver diff --git a/vital/types/video_uninterpreted_data.h b/vital/types/video_uninterpreted_data.h new file mode 100644 index 0000000000..76d94f3550 --- /dev/null +++ b/vital/types/video_uninterpreted_data.h @@ -0,0 +1,35 @@ +// This file is part of KWIVER, and is distributed under the +// OSI-approved BSD 3-Clause License. See top-level LICENSE file or +// https://github.com/Kitware/kwiver/blob/master/LICENSE for details. + +/// \file +/// Declaration of base video uninterpreted data type. + +#ifndef VITAL_VIDEO_UNINTERPRETED_DATA_H_ +#define VITAL_VIDEO_UNINTERPRETED_DATA_H_ + +#include + +#include + +namespace kwiver { + +namespace vital { + +// ---------------------------------------------------------------------------- +/// Base class for holding a single frame of uninterpreted data. +struct VITAL_EXPORT video_uninterpreted_data +{ + virtual ~video_uninterpreted_data(); +}; + +using video_uninterpreted_data_sptr = + std::shared_ptr< video_uninterpreted_data >; +using video_uninterpreted_data_uptr = + std::unique_ptr< video_uninterpreted_data >; + +} // namespace vital + +} // namespace kwiver + +#endif diff --git a/vital/util/text_codec_utf_8.cxx b/vital/util/text_codec_utf_8.cxx index 3f5c504050..69a4249851 100644 --- a/vital/util/text_codec_utf_8.cxx +++ b/vital/util/text_codec_utf_8.cxx @@ -9,6 +9,8 @@ #include +#include + namespace kwiver { namespace vital {