From 8e6aac6530320fc1215943f6f5e3069feaa75cb3 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Sat, 21 Jan 2023 13:44:49 -0800 Subject: [PATCH 01/65] WIP generating type hash during code gen Signed-off-by: Emerson Knapp --- rosidl_pycommon/rosidl_pycommon/__init__.py | 230 ++++++++++++++++++-- 1 file changed, 210 insertions(+), 20 deletions(-) diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 053d53c72..992dd4992 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib from io import StringIO import json import os @@ -21,6 +22,15 @@ import em from rosidl_parser.definition import IdlLocator +from rosidl_parser import definition +from rosidl_parser.definition import ( + IdlFile, + IdlContent, + Message, + Include, + Service, + Action +) from rosidl_parser.parser import parse_idl_file @@ -48,6 +58,148 @@ def get_newest_modification_time(target_dependencies): return newest_timestamp +# This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg +# TODO There is no FieldType.msg definition for the following rosidl_parser.definition types +# * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long +# * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long +# * +FIELD_TYPES = { + 'nested_type': 0, + 'int8': 1, + 'uint8': 2, + 'int16': 3, + 'uint16': 4, + 'int32': 5, + 'uint32': 6, + 'int64': 7, + 'uint64': 8, + 'float': 9, + 'double': 10, + 'long double': 11, + 'char': 12, + 'wchar': 13, + 'boolean': 14, + 'octet': 15, # byte + definition.UnboundedString: 16, + definition.UnboundedWString: 17, + # TODO there is no rosidl_parser.definition type for fixed strings (there is array of char, though?) + # FIXED_STRING = 18 + # FIXED_WSTRING = 19 + definition.BoundedString: 20, + definition.BoundedWString: 21, +} + +NESTED_FIELD_TYPE_OFFSETS = { + definition.Array: 32, + definition.BoundedSequence: 64, + definition.UnboundedSequence: 96, +} + + +def translate_type_id(value_type, offset, result): + if isinstance(value_type, definition.BasicType): + result['type_id'] = FIELD_TYPES[value_type.typename] + offset + elif isinstance(value_type, definition.AbstractGenericString): + result['type_id'] = FIELD_TYPES[type(value_type)] + offset + if value_type.has_maximum_size(): + result['string_length'] = value_type.maximum_size + elif isinstance(value_type, definition.NamespacedType): + result['type_id'] = offset + result['nested_type_name'] = '/'.join(value_type.namespaced_name()) + elif isinstance(value_type, definition.NamedType): + result['type_id'] = offset + result['nested_type_name'] = value_type.name + else: + raise TypeError('Unknown value type ', value_type) + + +def field_type(ftype: definition.AbstractType): + result = { + 'type_id': 0, + 'length': 0, + 'string_length': 0, + 'nested_type_name': '', + } + + if isinstance(ftype, definition.AbstractNestableType): + translate_type_id(ftype, 0, result) + elif isinstance(ftype, definition.AbstractNestedType): + type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] + translate_type_id(ftype.value_type, type_id_offset, result) + if ftype.has_maximum_size(): + try: + result['length'] = ftype.maximum_size + except AttributeError: + result['length'] = ftype.size + else: + print(ftype) + raise Exception('idk that type type!', ftype) + + return result + + +def field(member: definition.Member): + return { + 'name': member.name, + 'type': field_type(member.type), + # skipping default_value + } + + +def individual_type_description(msg: Message): + fields = [field(member) for member in msg.structure.members] + # referenced_types = [f['type']['nested_type_name'] for f in fields] + # referenced_types = [f for f in referenced_types if f != ''] + return { + 'type_name': '/'.join(msg.structure.namespaced_type.namespaced_name()), + 'fields': fields + } + + +def generate_type_version_hash(id_triplet, idl_files): + idl = idl_files[id_triplet] + + includes = [] + messages = [] + services = [] + actions = [] + for el in idl.content.elements: + if isinstance(el, Include): + includes.append(el) + print(f' Include: {el.locator}') + elif isinstance(el, Message): + messages.append(el) + print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') + elif isinstance(el, Service): + services.append(el) + print(f' Service: {el.namespaced_type.name}') + elif isinstance(el, Action): + actions.append(el) + print(el) + else: + raise Exception(f'Do not know how to hash {el}') + + # Per rosidl_parser.parser.extract_content_from_ast, + # IDL may have only one of Message, Service, or Action to be parsed + total_interfaces = len(messages) + len(services) + len(actions) + if total_interfaces < 1: + raise Exception('No interface defined in IDL, cannot hash') + if total_interfaces > 1: + raise Exception('More than one ROS interface defined in IDL, cannot hash') + + if len(messages): + serialization_data = { + 'type_description': individual_type_description(messages[0]), + 'referenced_type_descriptions': [], # TODO referenced type descriptions + } + # TODO remove indent + serialized_type_description = json.dumps(serialization_data, indent=2) + print(serialized_type_description) + m = hashlib.sha256() + m.update(serialized_type_description.encode('utf-8')) + return m.hexdigest() + + def generate_files( generator_arguments_file, mapping, additional_context=None, keep_case=False, post_process_callback=None @@ -62,39 +214,77 @@ def generate_files( latest_target_timestamp = get_newest_modification_time(args['target_dependencies']) generated_files = [] + # idl_locators = {} + idl_ids_to_generate = [] + idl_files = {} + package_name = args['package_name'] for idl_tuple in args.get('idl_tuples', []): idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 locator = IdlLocator(*idl_parts) + idl_rel_path = pathlib.Path(idl_parts[1]) + namespace = str(idl_rel_path.parent) idl_stem = idl_rel_path.stem - if not keep_case: - idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) + id_triplet = (package_name, namespace, idl_stem) try: - idl_file = parse_idl_file(locator) - for template_file, generated_filename in mapping.items(): - generated_file = os.path.join( - args['output_dir'], str(idl_rel_path.parent), - generated_filename % idl_stem) - generated_files.append(generated_file) - data = { - 'package_name': args['package_name'], - 'interface_path': idl_rel_path, - 'content': idl_file.content, - } - if additional_context is not None: - data.update(additional_context) - expand_template( - os.path.basename(template_file), data, - generated_file, minimum_timestamp=latest_target_timestamp, - template_basepath=template_basepath, - post_process_callback=post_process_callback) + print('Parsing ', id_triplet) + idl_files[id_triplet] = parse_idl_file(locator) + idl_ids_to_generate.append(id_triplet) except Exception as e: print( 'Error processing idl file: ' + str(locator.get_absolute_path()), file=sys.stderr) raise(e) + for interface_dep in args.get('ros_interface_dependencies', []): + tuple_parts = interface_dep.rsplit(':', 1) + assert len(tuple_parts) == 2 + package_name = tuple_parts[0] + idl_abs_path = tuple_parts[1] + + namespace = pathlib.Path(idl_abs_path).parents[0].name + idl_stem = pathlib.Path(idl_abs_path).stem + id_triplet = (package_name, namespace, idl_stem) + try: + print('Parsing ', id_triplet) + idl_files[id_triplet] = parse_idl_file(locator) + except Exception as e: + print( + 'Error processing idl file: ' + + str(locator.get_absolute_path()), file=sys.stderr) + raise(e) + + for id_triplet in idl_ids_to_generate: + _, _, idl_stem = id_triplet + if not keep_case: + idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) + + print(id_triplet) + type_hash = generate_type_version_hash(id_triplet, idl_files) + print(type_hash) + + idl_file = idl_files[id_triplet] + raise Exception('poop') + for template_file, generated_filename in mapping.items(): + generated_file = os.path.join( + args['output_dir'], str(idl_rel_path.parent), + generated_filename % idl_stem) + generated_files.append(generated_file) + data = { + 'package_name': package_name, + 'interface_path': idl_rel_path, + 'content': idl_file.content, + 'type_hash': type_hash, + } + if additional_context is not None: + data.update(additional_context) + expand_template( + os.path.basename(template_file), data, + generated_file, minimum_timestamp=latest_target_timestamp, + template_basepath=template_basepath, + post_process_callback=post_process_callback) + return generated_files From cd8e81c2c088ed5e31a14c4810d3f775fb391569 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Sun, 22 Jan 2023 11:19:06 -0800 Subject: [PATCH 02/65] WIP starting to look at referenced types as well Signed-off-by: Emerson Knapp --- rosidl_pycommon/rosidl_pycommon/__init__.py | 35 +++++++++------------ 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 992dd4992..b3eb08d90 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -21,16 +21,7 @@ import sys import em -from rosidl_parser.definition import IdlLocator -from rosidl_parser import definition -from rosidl_parser.definition import ( - IdlFile, - IdlContent, - Message, - Include, - Service, - Action -) +from rosidl_parser import definition from rosidl_parser.parser import parse_idl_file @@ -146,7 +137,7 @@ def field(member: definition.Member): } -def individual_type_description(msg: Message): +def individual_type_description(msg: definition.Message): fields = [field(member) for member in msg.structure.members] # referenced_types = [f['type']['nested_type_name'] for f in fields] # referenced_types = [f for f in referenced_types if f != ''] @@ -164,16 +155,16 @@ def generate_type_version_hash(id_triplet, idl_files): services = [] actions = [] for el in idl.content.elements: - if isinstance(el, Include): + if isinstance(el, definition.Include): includes.append(el) print(f' Include: {el.locator}') - elif isinstance(el, Message): + elif isinstance(el, definition.Message): messages.append(el) print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') - elif isinstance(el, Service): + elif isinstance(el, definition.Service): services.append(el) print(f' Service: {el.namespaced_type.name}') - elif isinstance(el, Action): + elif isinstance(el, definition.Action): actions.append(el) print(el) else: @@ -221,10 +212,12 @@ def generate_files( for idl_tuple in args.get('idl_tuples', []): idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 - locator = IdlLocator(*idl_parts) + print(idl_parts) + locator = definition.IdlLocator(*idl_parts) idl_rel_path = pathlib.Path(idl_parts[1]) namespace = str(idl_rel_path.parent) + print(idl_rel_path) idl_stem = idl_rel_path.stem id_triplet = (package_name, namespace, idl_stem) try: @@ -240,12 +233,15 @@ def generate_files( for interface_dep in args.get('ros_interface_dependencies', []): tuple_parts = interface_dep.rsplit(':', 1) assert len(tuple_parts) == 2 - package_name = tuple_parts[0] - idl_abs_path = tuple_parts[1] + print(tuple_parts) + referenced_package_name, idl_abs_path = tuple_parts + + base_path, sep, rel_path = idl_abs_path.rpartition(referenced_package_name) + locator = definition.IdlLocator(idl_abs_path, '') namespace = pathlib.Path(idl_abs_path).parents[0].name idl_stem = pathlib.Path(idl_abs_path).stem - id_triplet = (package_name, namespace, idl_stem) + id_triplet = (referenced_package_name, namespace, idl_stem) try: print('Parsing ', id_triplet) idl_files[id_triplet] = parse_idl_file(locator) @@ -265,7 +261,6 @@ def generate_files( print(type_hash) idl_file = idl_files[id_triplet] - raise Exception('poop') for template_file, generated_filename in mapping.items(): generated_file = os.path.join( args['output_dir'], str(idl_rel_path.parent), From ce77ab1867c21149906902966596c452ac2306c1 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 23 Jan 2023 12:17:41 -0800 Subject: [PATCH 03/65] Refactoring type hash to other file Signed-off-by: Emerson Knapp --- rosidl_parser/rosidl_parser/type_hash.py | 157 +++++++++++++++++++ rosidl_pycommon/rosidl_pycommon/__init__.py | 162 ++------------------ 2 files changed, 168 insertions(+), 151 deletions(-) create mode 100644 rosidl_parser/rosidl_parser/type_hash.py diff --git a/rosidl_parser/rosidl_parser/type_hash.py b/rosidl_parser/rosidl_parser/type_hash.py new file mode 100644 index 000000000..ed0b78885 --- /dev/null +++ b/rosidl_parser/rosidl_parser/type_hash.py @@ -0,0 +1,157 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json + +from rosidl_parser import definition + +# This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg +# TODO There is no FieldType.msg definition for the following rosidl_parser.definition types +# * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long +# * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long +# * +FIELD_TYPES = { + 'nested_type': 0, + 'int8': 1, + 'uint8': 2, + 'int16': 3, + 'uint16': 4, + 'int32': 5, + 'uint32': 6, + 'int64': 7, + 'uint64': 8, + 'float': 9, + 'double': 10, + 'long double': 11, + 'char': 12, + 'wchar': 13, + 'boolean': 14, + 'octet': 15, # byte + definition.UnboundedString: 16, + definition.UnboundedWString: 17, + # TODO there is no rosidl_parser.definition type for fixed strings (there is array of char, though?) + # FIXED_STRING = 18 + # FIXED_WSTRING = 19 + definition.BoundedString: 20, + definition.BoundedWString: 21, +} + +NESTED_FIELD_TYPE_OFFSETS = { + definition.Array: 32, + definition.BoundedSequence: 64, + definition.UnboundedSequence: 96, +} + +def translate_type_id(value_type, offset, result): + if isinstance(value_type, definition.BasicType): + result['type_id'] = FIELD_TYPES[value_type.typename] + offset + elif isinstance(value_type, definition.AbstractGenericString): + result['type_id'] = FIELD_TYPES[type(value_type)] + offset + if value_type.has_maximum_size(): + result['string_length'] = value_type.maximum_size + elif isinstance(value_type, definition.NamespacedType): + result['type_id'] = offset + result['nested_type_name'] = '/'.join(value_type.namespaced_name()) + elif isinstance(value_type, definition.NamedType): + result['type_id'] = offset + result['nested_type_name'] = value_type.name + else: + raise TypeError('Unknown value type ', value_type) + + +def serialize_field_type(ftype: definition.AbstractType): + result = { + 'type_id': 0, + 'length': 0, + 'string_length': 0, + 'nested_type_name': '', + } + + if isinstance(ftype, definition.AbstractNestableType): + translate_type_id(ftype, 0, result) + elif isinstance(ftype, definition.AbstractNestedType): + type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] + translate_type_id(ftype.value_type, type_id_offset, result) + if ftype.has_maximum_size(): + try: + result['length'] = ftype.maximum_size + except AttributeError: + result['length'] = ftype.size + else: + print(ftype) + raise Exception('Unable to translate field type', ftype) + + return result + + +def serialize_field(member: definition.Member): + return { + 'name': member.name, + 'type': serialize_field_type(member.type), + # skipping default_value + } + + +def serialize_individual_type_description(msg: definition.Message): + fields = [serialize_field(member) for member in msg.structure.members] + # referenced_types = [f['type']['nested_type_name'] for f in fields] + # referenced_types = [f for f in referenced_types if f != ''] + return { + 'type_name': '/'.join(msg.structure.namespaced_type.namespaced_name()), + 'fields': fields + } + +def generate_type_version_hash(id_triplet, idl_files): + idl = idl_files[id_triplet] + + includes = [] + messages = [] + services = [] + actions = [] + for el in idl.content.elements: + if isinstance(el, definition.Include): + includes.append(el) + print(f' Include: {el.locator}') + elif isinstance(el, definition.Message): + messages.append(el) + print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') + elif isinstance(el, definition.Service): + services.append(el) + print(f' Service: {el.namespaced_type.name}') + elif isinstance(el, definition.Action): + actions.append(el) + print(el) + else: + raise Exception(f'Do not know how to hash {el}') + + # Per rosidl_parser.parser.extract_content_from_ast, + # IDL may have only one of Message, Service, or Action to be parsed + total_interfaces = len(messages) + len(services) + len(actions) + if total_interfaces < 1: + raise Exception('No interface defined in IDL, cannot hash') + if total_interfaces > 1: + raise Exception('More than one ROS interface defined in IDL, cannot hash') + + if len(messages): + serialization_data = { + 'type_description': serialize_individual_type_description(messages[0]), + 'referenced_type_descriptions': [], # TODO referenced type descriptions + } + # TODO remove indent + serialized_type_description = json.dumps(serialization_data, indent=2) + print(serialized_type_description) + m = hashlib.sha256() + m.update(serialized_type_description.encode('utf-8')) + return m.hexdigest() diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index b3eb08d90..adf692050 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -21,8 +21,9 @@ import sys import em -from rosidl_parser import definition +from rosidl_parser.definition import IdlLocator from rosidl_parser.parser import parse_idl_file +from rosidl_parser.type_hash import generate_type_version_hash def convert_camel_case_to_lower_case_underscore(value): @@ -49,148 +50,6 @@ def get_newest_modification_time(target_dependencies): return newest_timestamp -# This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg -# TODO There is no FieldType.msg definition for the following rosidl_parser.definition types -# * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long -# * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long -# * -FIELD_TYPES = { - 'nested_type': 0, - 'int8': 1, - 'uint8': 2, - 'int16': 3, - 'uint16': 4, - 'int32': 5, - 'uint32': 6, - 'int64': 7, - 'uint64': 8, - 'float': 9, - 'double': 10, - 'long double': 11, - 'char': 12, - 'wchar': 13, - 'boolean': 14, - 'octet': 15, # byte - definition.UnboundedString: 16, - definition.UnboundedWString: 17, - # TODO there is no rosidl_parser.definition type for fixed strings (there is array of char, though?) - # FIXED_STRING = 18 - # FIXED_WSTRING = 19 - definition.BoundedString: 20, - definition.BoundedWString: 21, -} - -NESTED_FIELD_TYPE_OFFSETS = { - definition.Array: 32, - definition.BoundedSequence: 64, - definition.UnboundedSequence: 96, -} - - -def translate_type_id(value_type, offset, result): - if isinstance(value_type, definition.BasicType): - result['type_id'] = FIELD_TYPES[value_type.typename] + offset - elif isinstance(value_type, definition.AbstractGenericString): - result['type_id'] = FIELD_TYPES[type(value_type)] + offset - if value_type.has_maximum_size(): - result['string_length'] = value_type.maximum_size - elif isinstance(value_type, definition.NamespacedType): - result['type_id'] = offset - result['nested_type_name'] = '/'.join(value_type.namespaced_name()) - elif isinstance(value_type, definition.NamedType): - result['type_id'] = offset - result['nested_type_name'] = value_type.name - else: - raise TypeError('Unknown value type ', value_type) - - -def field_type(ftype: definition.AbstractType): - result = { - 'type_id': 0, - 'length': 0, - 'string_length': 0, - 'nested_type_name': '', - } - - if isinstance(ftype, definition.AbstractNestableType): - translate_type_id(ftype, 0, result) - elif isinstance(ftype, definition.AbstractNestedType): - type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] - translate_type_id(ftype.value_type, type_id_offset, result) - if ftype.has_maximum_size(): - try: - result['length'] = ftype.maximum_size - except AttributeError: - result['length'] = ftype.size - else: - print(ftype) - raise Exception('idk that type type!', ftype) - - return result - - -def field(member: definition.Member): - return { - 'name': member.name, - 'type': field_type(member.type), - # skipping default_value - } - - -def individual_type_description(msg: definition.Message): - fields = [field(member) for member in msg.structure.members] - # referenced_types = [f['type']['nested_type_name'] for f in fields] - # referenced_types = [f for f in referenced_types if f != ''] - return { - 'type_name': '/'.join(msg.structure.namespaced_type.namespaced_name()), - 'fields': fields - } - - -def generate_type_version_hash(id_triplet, idl_files): - idl = idl_files[id_triplet] - - includes = [] - messages = [] - services = [] - actions = [] - for el in idl.content.elements: - if isinstance(el, definition.Include): - includes.append(el) - print(f' Include: {el.locator}') - elif isinstance(el, definition.Message): - messages.append(el) - print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') - elif isinstance(el, definition.Service): - services.append(el) - print(f' Service: {el.namespaced_type.name}') - elif isinstance(el, definition.Action): - actions.append(el) - print(el) - else: - raise Exception(f'Do not know how to hash {el}') - - # Per rosidl_parser.parser.extract_content_from_ast, - # IDL may have only one of Message, Service, or Action to be parsed - total_interfaces = len(messages) + len(services) + len(actions) - if total_interfaces < 1: - raise Exception('No interface defined in IDL, cannot hash') - if total_interfaces > 1: - raise Exception('More than one ROS interface defined in IDL, cannot hash') - - if len(messages): - serialization_data = { - 'type_description': individual_type_description(messages[0]), - 'referenced_type_descriptions': [], # TODO referenced type descriptions - } - # TODO remove indent - serialized_type_description = json.dumps(serialization_data, indent=2) - print(serialized_type_description) - m = hashlib.sha256() - m.update(serialized_type_description.encode('utf-8')) - return m.hexdigest() - - def generate_files( generator_arguments_file, mapping, additional_context=None, keep_case=False, post_process_callback=None @@ -212,16 +71,15 @@ def generate_files( for idl_tuple in args.get('idl_tuples', []): idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 + locator = IdlLocator(*idl_parts) + print("---Local") print(idl_parts) - locator = definition.IdlLocator(*idl_parts) idl_rel_path = pathlib.Path(idl_parts[1]) namespace = str(idl_rel_path.parent) - print(idl_rel_path) idl_stem = idl_rel_path.stem id_triplet = (package_name, namespace, idl_stem) try: - print('Parsing ', id_triplet) idl_files[id_triplet] = parse_idl_file(locator) idl_ids_to_generate.append(id_triplet) except Exception as e: @@ -233,17 +91,18 @@ def generate_files( for interface_dep in args.get('ros_interface_dependencies', []): tuple_parts = interface_dep.rsplit(':', 1) assert len(tuple_parts) == 2 - print(tuple_parts) referenced_package_name, idl_abs_path = tuple_parts base_path, sep, rel_path = idl_abs_path.rpartition(referenced_package_name) - locator = definition.IdlLocator(idl_abs_path, '') + print("---Dependency") + print(referenced_package_name, rel_path) + assert sep == referenced_package_name + locator = IdlLocator(idl_abs_path, '') namespace = pathlib.Path(idl_abs_path).parents[0].name idl_stem = pathlib.Path(idl_abs_path).stem id_triplet = (referenced_package_name, namespace, idl_stem) try: - print('Parsing ', id_triplet) idl_files[id_triplet] = parse_idl_file(locator) except Exception as e: print( @@ -251,14 +110,15 @@ def generate_files( str(locator.get_absolute_path()), file=sys.stderr) raise(e) + print(idl_files) for id_triplet in idl_ids_to_generate: _, _, idl_stem = id_triplet if not keep_case: idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) - print(id_triplet) type_hash = generate_type_version_hash(id_triplet, idl_files) print(type_hash) + raise Exception('poop') idl_file = idl_files[id_triplet] for template_file, generated_filename in mapping.items(): @@ -270,7 +130,7 @@ def generate_files( 'package_name': package_name, 'interface_path': idl_rel_path, 'content': idl_file.content, - 'type_hash': type_hash, + # 'type_hash': type_hash, } if additional_context is not None: data.update(additional_context) From 6d582eaebb1a40d939718a084f9280005004421f Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 23 Jan 2023 13:55:19 -0800 Subject: [PATCH 04/65] Including referenced type desrciptions now Signed-off-by: Emerson Knapp --- rosidl_parser/rosidl_parser/type_hash.py | 59 +++++++++++---------- rosidl_pycommon/rosidl_pycommon/__init__.py | 43 ++++++--------- 2 files changed, 45 insertions(+), 57 deletions(-) diff --git a/rosidl_parser/rosidl_parser/type_hash.py b/rosidl_parser/rosidl_parser/type_hash.py index ed0b78885..0f548151f 100644 --- a/rosidl_parser/rosidl_parser/type_hash.py +++ b/rosidl_parser/rosidl_parser/type_hash.py @@ -113,45 +113,46 @@ def serialize_individual_type_description(msg: definition.Message): 'fields': fields } -def generate_type_version_hash(id_triplet, idl_files): - idl = idl_files[id_triplet] + +def generate_type_version_hash(file_key, idl_files): + idl = idl_files[file_key] includes = [] - messages = [] - services = [] - actions = [] + referenced_type_descriptions = {} + serialization_data = { + 'type_description': None, + 'referenced_type_descriptions': [], + } + for el in idl.content.elements: if isinstance(el, definition.Include): - includes.append(el) + includes.append(el.locator) print(f' Include: {el.locator}') elif isinstance(el, definition.Message): - messages.append(el) print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') + serialization_data['type_description'] = serialize_individual_type_description(el) elif isinstance(el, definition.Service): - services.append(el) print(f' Service: {el.namespaced_type.name}') elif isinstance(el, definition.Action): - actions.append(el) - print(el) + print(f' Action: {el}') else: raise Exception(f'Do not know how to hash {el}') - # Per rosidl_parser.parser.extract_content_from_ast, - # IDL may have only one of Message, Service, or Action to be parsed - total_interfaces = len(messages) + len(services) + len(actions) - if total_interfaces < 1: - raise Exception('No interface defined in IDL, cannot hash') - if total_interfaces > 1: - raise Exception('More than one ROS interface defined in IDL, cannot hash') - - if len(messages): - serialization_data = { - 'type_description': serialize_individual_type_description(messages[0]), - 'referenced_type_descriptions': [], # TODO referenced type descriptions - } - # TODO remove indent - serialized_type_description = json.dumps(serialization_data, indent=2) - print(serialized_type_description) - m = hashlib.sha256() - m.update(serialized_type_description.encode('utf-8')) - return m.hexdigest() + while includes: + locator = includes.pop() + if locator not in referenced_type_descriptions: + included_file = idl_files[locator] + for el in included_file.content.elements: + if isinstance(el, definition.Include): + includes.append(el.locator) + elif isinstance(el, definition.Message): + referenced_type_descriptions[locator] = serialize_individual_type_description(el) + + referenced_type_descriptions + serialization_data['referenced_type_descriptions'] = sorted( + referenced_type_descriptions.items(), key=lambda td: td['type_name']) + serialized_type_description = json.dumps(serialization_data) + # print(serialized_type_description) + m = hashlib.sha256() + m.update(serialized_type_description.encode('utf-8')) + return m.hexdigest() diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index adf692050..09f39d3ee 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -64,24 +64,18 @@ def generate_files( latest_target_timestamp = get_newest_modification_time(args['target_dependencies']) generated_files = [] - # idl_locators = {} - idl_ids_to_generate = [] + idl_files_to_generate = [] idl_files = {} package_name = args['package_name'] for idl_tuple in args.get('idl_tuples', []): idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 - locator = IdlLocator(*idl_parts) - print("---Local") - print(idl_parts) + namespaced_idl_path = str(pathlib.Path(package_name) / idl_parts[1]) - idl_rel_path = pathlib.Path(idl_parts[1]) - namespace = str(idl_rel_path.parent) - idl_stem = idl_rel_path.stem - id_triplet = (package_name, namespace, idl_stem) + locator = IdlLocator(*idl_parts) try: - idl_files[id_triplet] = parse_idl_file(locator) - idl_ids_to_generate.append(id_triplet) + idl_files[namespaced_idl_path] = parse_idl_file(locator) + idl_files_to_generate.append(namespaced_idl_path) except Exception as e: print( 'Error processing idl file: ' + @@ -93,34 +87,27 @@ def generate_files( assert len(tuple_parts) == 2 referenced_package_name, idl_abs_path = tuple_parts - base_path, sep, rel_path = idl_abs_path.rpartition(referenced_package_name) - print("---Dependency") - print(referenced_package_name, rel_path) - assert sep == referenced_package_name - locator = IdlLocator(idl_abs_path, '') + base_path, pkg, rel_path = idl_abs_path.rpartition(referenced_package_name) + assert pkg == referenced_package_name + namespaced_idl_path = pkg + rel_path - namespace = pathlib.Path(idl_abs_path).parents[0].name - idl_stem = pathlib.Path(idl_abs_path).stem - id_triplet = (referenced_package_name, namespace, idl_stem) + # rel_path it starts with a pathsep from rpartition + locator = IdlLocator(base_path + pkg, rel_path[1:]) try: - idl_files[id_triplet] = parse_idl_file(locator) + idl_files[namespaced_idl_path] = parse_idl_file(locator) except Exception as e: print( 'Error processing idl file: ' + str(locator.get_absolute_path()), file=sys.stderr) raise(e) - print(idl_files) - for id_triplet in idl_ids_to_generate: - _, _, idl_stem = id_triplet + for file_key in idl_files_to_generate: + idl_stem = pathlib.Path(file_key).stem if not keep_case: idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) + type_hash = generate_type_version_hash(file_key, idl_files) - type_hash = generate_type_version_hash(id_triplet, idl_files) - print(type_hash) - raise Exception('poop') - - idl_file = idl_files[id_triplet] + idl_file = idl_files[file_key] for template_file, generated_filename in mapping.items(): generated_file = os.path.join( args['output_dir'], str(idl_rel_path.parent), From a51bf43b40ca1bfef7f1859d13d856ec947dc827 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 23 Jan 2023 22:04:02 -0800 Subject: [PATCH 05/65] Added type version hash to all 3 generators Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 28 ++++++++++--------- rosidl_generator_c/resource/msg__struct.h.em | 8 ++++++ .../resource/idl__struct.hpp.em | 6 ++-- .../resource/msg__struct.hpp.em | 9 ++++++ .../resource/srv__struct.hpp.em | 4 +-- rosidl_parser/rosidl_parser/type_hash.py | 19 +++++++++---- rosidl_pycommon/rosidl_pycommon/__init__.py | 11 ++++++-- 7 files changed, 58 insertions(+), 27 deletions(-) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index 94bf39bf4..5fb9ebc85 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -32,6 +32,7 @@ extern "C" #include #include + @####################################################################### @# Handle message @####################################################################### @@ -43,7 +44,7 @@ from rosidl_parser.definition import Message TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=message, include_directives=include_directives) + message=message, include_directives=include_directives, type_hash=type_hash) }@ @[end for]@ @@ -59,21 +60,21 @@ from rosidl_parser.definition import Service TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=service.request_message, include_directives=include_directives) + message=service.request_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=service.response_message, include_directives=include_directives) + message=service.response_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=service.event_message, include_directives=include_directives) + message=service.event_message, include_directives=include_directives, type_hash=type_hash) }@ @[end for]@ @@ -89,63 +90,63 @@ from rosidl_parser.definition import Action TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.goal, include_directives=include_directives) + message=action.goal, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.result, include_directives=include_directives) + message=action.result, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.feedback, include_directives=include_directives) + message=action.feedback, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.send_goal_service.request_message, include_directives=include_directives) + message=action.send_goal_service.request_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.send_goal_service.response_message, include_directives=include_directives) + message=action.send_goal_service.response_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.send_goal_service.event_message, include_directives=include_directives) + message=action.send_goal_service.event_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.get_result_service.request_message, include_directives=include_directives) + message=action.get_result_service.request_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.get_result_service.response_message, include_directives=include_directives) + message=action.get_result_service.response_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.get_result_service.event_message, include_directives=include_directives) + message=action.get_result_service.event_message, include_directives=include_directives, type_hash=type_hash) }@ @{ @@ -153,6 +154,7 @@ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, message=action.feedback_message, include_directives=include_directives) + message=action.feedback_message, include_directives=include_directives, type_hash=type_hash) }@ @[end for]@ diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index f0b99e166..2a111c936 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -61,6 +61,14 @@ for member in message.structure.members: }@ @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +// Type Version Hash for interface +static const uint8_t @(idl_structure_type_to_c_typename(message.structure.namespaced_type))__TYPE_VERSION_HASH[32] = { + @ +@[for i in range(32)]@ +0x@(type_hash[i:i+1].hex()), @ +@[end for] +}; + @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< // Constants defined in the message @[for constant in message.constants]@ diff --git a/rosidl_generator_cpp/resource/idl__struct.hpp.em b/rosidl_generator_cpp/resource/idl__struct.hpp.em index e29a1f89e..becc1d439 100644 --- a/rosidl_generator_cpp/resource/idl__struct.hpp.em +++ b/rosidl_generator_cpp/resource/idl__struct.hpp.em @@ -43,7 +43,7 @@ from rosidl_parser.definition import Message TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=message, include_directives=include_directives) + message=message, include_directives=include_directives, type_hash=type_hash) }@ @[end for]@ @@ -59,7 +59,7 @@ from rosidl_parser.definition import Service TEMPLATE( 'srv__struct.hpp.em', package_name=package_name, interface_path=interface_path, service=service, - include_directives=include_directives) + include_directives=include_directives, type_hash=type_hash) }@ @[end for]@ @@ -75,7 +75,7 @@ from rosidl_parser.definition import Action TEMPLATE( 'action__struct.hpp.em', package_name=package_name, interface_path=interface_path, action=action, - include_directives=include_directives) + include_directives=include_directives, type_hash=type_hash) }@ @[end for]@ diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index 02097f15e..2db39d23c 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -101,6 +101,15 @@ struct @(message.structure.namespaced_type.name)_ { using Type = @(message.structure.namespaced_type.name)_; + + // Type Version Hash for interface + static constexpr uint8_t TYPE_VERSION_HASH[32] = { + @ +@[for i in range(32)]@ +0x@(type_hash[i:i+1].hex()), @ +@[end for] + }; + @{ # The creation of the constructors for messages is a bit complicated. The goal # is to have a constructor where the user can control how the fields of the diff --git a/rosidl_generator_cpp/resource/srv__struct.hpp.em b/rosidl_generator_cpp/resource/srv__struct.hpp.em index 1466c9980..c22618b30 100644 --- a/rosidl_generator_cpp/resource/srv__struct.hpp.em +++ b/rosidl_generator_cpp/resource/srv__struct.hpp.em @@ -3,14 +3,14 @@ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=service.request_message, include_directives=include_directives) + message=service.request_message, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=service.response_message, include_directives=include_directives) + message=service.response_message, include_directives=include_directives, type_hash=type_hash) }@ @{ diff --git a/rosidl_parser/rosidl_parser/type_hash.py b/rosidl_parser/rosidl_parser/type_hash.py index 0f548151f..0e71978e7 100644 --- a/rosidl_parser/rosidl_parser/type_hash.py +++ b/rosidl_parser/rosidl_parser/type_hash.py @@ -127,14 +127,20 @@ def generate_type_version_hash(file_key, idl_files): for el in idl.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) - print(f' Include: {el.locator}') + # print(f' Include: {el.locator}') elif isinstance(el, definition.Message): - print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') + # print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') serialization_data['type_description'] = serialize_individual_type_description(el) elif isinstance(el, definition.Service): - print(f' Service: {el.namespaced_type.name}') + serialization_data['type_description'] = { + 'request_message': serialize_individual_type_description(el.request_message), + 'response_message': serialize_individual_type_description(el.response_message), + } + # print(f' Service: {el.namespaced_type.name}') + pass elif isinstance(el, definition.Action): - print(f' Action: {el}') + # print(f' Action: {el}') + pass else: raise Exception(f'Do not know how to hash {el}') @@ -145,14 +151,15 @@ def generate_type_version_hash(file_key, idl_files): for el in included_file.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) + # print(f' Include: {el.locator}') elif isinstance(el, definition.Message): referenced_type_descriptions[locator] = serialize_individual_type_description(el) referenced_type_descriptions serialization_data['referenced_type_descriptions'] = sorted( - referenced_type_descriptions.items(), key=lambda td: td['type_name']) + referenced_type_descriptions.values(), key=lambda td: td['type_name']) serialized_type_description = json.dumps(serialization_data) # print(serialized_type_description) m = hashlib.sha256() m.update(serialized_type_description.encode('utf-8')) - return m.hexdigest() + return m.digest() diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 09f39d3ee..7f18629b9 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -67,6 +67,7 @@ def generate_files( idl_files_to_generate = [] idl_files = {} package_name = args['package_name'] + for idl_tuple in args.get('idl_tuples', []): idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 @@ -102,10 +103,14 @@ def generate_files( raise(e) for file_key in idl_files_to_generate: - idl_stem = pathlib.Path(file_key).stem + idl_rel_path = pathlib.Path(file_key) + idl_rel_path = idl_rel_path.relative_to(idl_rel_path.parts[0]) + idl_stem = idl_rel_path.stem + type_hash = generate_type_version_hash(file_key, idl_files) + print(f'{file_key}: {type_hash}') + if not keep_case: idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) - type_hash = generate_type_version_hash(file_key, idl_files) idl_file = idl_files[file_key] for template_file, generated_filename in mapping.items(): @@ -117,7 +122,7 @@ def generate_files( 'package_name': package_name, 'interface_path': idl_rel_path, 'content': idl_file.content, - # 'type_hash': type_hash, + 'type_hash': type_hash, } if additional_context is not None: data.update(additional_context) From e37e8e561d3a364633656a05fb0eed19794b5072 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 23 Jan 2023 22:54:39 -0800 Subject: [PATCH 06/65] Minor cleanup for readability Signed-off-by: Emerson Knapp --- rosidl_parser/rosidl_parser/type_hash.py | 14 +++----------- rosidl_pycommon/rosidl_pycommon/__init__.py | 1 - 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/rosidl_parser/rosidl_parser/type_hash.py b/rosidl_parser/rosidl_parser/type_hash.py index 0e71978e7..3a7a8f1f6 100644 --- a/rosidl_parser/rosidl_parser/type_hash.py +++ b/rosidl_parser/rosidl_parser/type_hash.py @@ -90,7 +90,6 @@ def serialize_field_type(ftype: definition.AbstractType): except AttributeError: result['length'] = ftype.size else: - print(ftype) raise Exception('Unable to translate field type', ftype) return result @@ -105,12 +104,9 @@ def serialize_field(member: definition.Member): def serialize_individual_type_description(msg: definition.Message): - fields = [serialize_field(member) for member in msg.structure.members] - # referenced_types = [f['type']['nested_type_name'] for f in fields] - # referenced_types = [f for f in referenced_types if f != ''] return { 'type_name': '/'.join(msg.structure.namespaced_type.namespaced_name()), - 'fields': fields + 'fields': [serialize_field(member) for member in msg.structure.members] } @@ -127,19 +123,16 @@ def generate_type_version_hash(file_key, idl_files): for el in idl.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) - # print(f' Include: {el.locator}') elif isinstance(el, definition.Message): - # print(f' Message: {el.structure.namespaced_type.namespaces} / {el.structure.namespaced_type.name}') serialization_data['type_description'] = serialize_individual_type_description(el) elif isinstance(el, definition.Service): serialization_data['type_description'] = { 'request_message': serialize_individual_type_description(el.request_message), 'response_message': serialize_individual_type_description(el.response_message), } - # print(f' Service: {el.namespaced_type.name}') pass elif isinstance(el, definition.Action): - # print(f' Action: {el}') + # TODO pass else: raise Exception(f'Do not know how to hash {el}') @@ -151,7 +144,6 @@ def generate_type_version_hash(file_key, idl_files): for el in included_file.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) - # print(f' Include: {el.locator}') elif isinstance(el, definition.Message): referenced_type_descriptions[locator] = serialize_individual_type_description(el) @@ -159,7 +151,7 @@ def generate_type_version_hash(file_key, idl_files): serialization_data['referenced_type_descriptions'] = sorted( referenced_type_descriptions.values(), key=lambda td: td['type_name']) serialized_type_description = json.dumps(serialization_data) - # print(serialized_type_description) + # print(json.dumps(serialization_data, indent=2)) m = hashlib.sha256() m.update(serialized_type_description.encode('utf-8')) return m.digest() diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 7f18629b9..cad18891a 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -107,7 +107,6 @@ def generate_files( idl_rel_path = idl_rel_path.relative_to(idl_rel_path.parts[0]) idl_stem = idl_rel_path.stem type_hash = generate_type_version_hash(file_key, idl_files) - print(f'{file_key}: {type_hash}') if not keep_case: idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) From 4834bfaf5e83ac2f92349f41f850f86a1abeb584 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 26 Jan 2023 21:09:51 -0800 Subject: [PATCH 07/65] Output a json file (for every generator, want to fix this) Signed-off-by: Emerson Knapp --- .../resource/action__struct.hpp.em | 12 ++++++------ rosidl_parser/rosidl_parser/type_hash.py | 8 ++++++-- rosidl_pycommon/rosidl_pycommon/__init__.py | 17 +++++++++++++++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index 020455074..6d1a3f12e 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -17,42 +17,42 @@ action_name = '::'.join(action.namespaced_type.namespaced_name()) TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.goal, include_directives=include_directives) + message=action.goal, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.result, include_directives=include_directives) + message=action.result, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.feedback, include_directives=include_directives) + message=action.feedback, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'srv__struct.hpp.em', package_name=package_name, interface_path=interface_path, - service=action.send_goal_service, include_directives=include_directives) + service=action.send_goal_service, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'srv__struct.hpp.em', package_name=package_name, interface_path=interface_path, - service=action.get_result_service, include_directives=include_directives) + service=action.get_result_service, include_directives=include_directives, type_hash=type_hash) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.feedback_message, include_directives=include_directives) + message=action.feedback_message, include_directives=include_directives, type_hash=type_hash) }@ @[for header_file in action_includes]@ diff --git a/rosidl_parser/rosidl_parser/type_hash.py b/rosidl_parser/rosidl_parser/type_hash.py index 3a7a8f1f6..fb5a5c308 100644 --- a/rosidl_parser/rosidl_parser/type_hash.py +++ b/rosidl_parser/rosidl_parser/type_hash.py @@ -110,7 +110,7 @@ def serialize_individual_type_description(msg: definition.Message): } -def generate_type_version_hash(file_key, idl_files): +def idl_to_hashable_json(file_key, idl_files): idl = idl_files[file_key] includes = [] @@ -150,7 +150,11 @@ def generate_type_version_hash(file_key, idl_files): referenced_type_descriptions serialization_data['referenced_type_descriptions'] = sorted( referenced_type_descriptions.values(), key=lambda td: td['type_name']) - serialized_type_description = json.dumps(serialization_data) + return json.dumps(serialization_data) + + +def generate_type_version_hash(file_key, idl_files): + serialized_type_description = idl_to_hashable_json(file_key, idl_files) # print(json.dumps(serialization_data, indent=2)) m = hashlib.sha256() m.update(serialized_type_description.encode('utf-8')) diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index cad18891a..0554a6c5a 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -24,6 +24,7 @@ from rosidl_parser.definition import IdlLocator from rosidl_parser.parser import parse_idl_file from rosidl_parser.type_hash import generate_type_version_hash +from rosidl_parser.type_hash import idl_to_hashable_json def convert_camel_case_to_lower_case_underscore(value): @@ -106,11 +107,23 @@ def generate_files( idl_rel_path = pathlib.Path(file_key) idl_rel_path = idl_rel_path.relative_to(idl_rel_path.parts[0]) idl_stem = idl_rel_path.stem - type_hash = generate_type_version_hash(file_key, idl_files) - if not keep_case: idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) + # Generate hashable representation and write to file + json_repr = idl_to_hashable_json(file_key, idl_files) + # TODO(emersonknapp) helper fn? + json_file = pathlib.Path(args['output_dir']) / idl_rel_path.parent / f'{idl_stem}.json' + json_file.parent.mkdir(parents=True, exist_ok=True) + with json_file.open('w', encoding='utf-8') as f: + f.write(json_repr) + + # Create hash + m = hashlib.sha256() + m.update(json_repr.encode('utf-8')) + type_hash = m.digest() + + # Run codegen for files idl_file = idl_files[file_key] for template_file, generated_filename in mapping.items(): generated_file = os.path.join( From edc61c2745e1544c3cd57b9f0d112cabde53d32b Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 2 Feb 2023 10:56:17 -0800 Subject: [PATCH 08/65] WIP just copying generator C feeling out what I really need for this generator Signed-off-by: Emerson Knapp --- rosidl_generator_type_hash/CHANGELOG.rst | 141 +++++++++++ rosidl_generator_type_hash/CMakeLists.txt | 25 ++ .../bin/rosidl_generator_type_hash | 38 +++ ...erator_type_hash_generate_interfaces.cmake | 200 ++++++++++++++++ rosidl_generator_type_hash/package.xml | 34 +++ .../rosidl_generator_type_hash/__init__.py | 220 ++++++++++++++++++ .../rosidl_generator_type_hash/cli.py | 81 +++++++ rosidl_generator_type_hash/setup.cfg | 3 + 8 files changed, 742 insertions(+) create mode 100644 rosidl_generator_type_hash/CHANGELOG.rst create mode 100644 rosidl_generator_type_hash/CMakeLists.txt create mode 100755 rosidl_generator_type_hash/bin/rosidl_generator_type_hash create mode 100644 rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake create mode 100644 rosidl_generator_type_hash/package.xml create mode 100644 rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py create mode 100644 rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py create mode 100644 rosidl_generator_type_hash/setup.cfg diff --git a/rosidl_generator_type_hash/CHANGELOG.rst b/rosidl_generator_type_hash/CHANGELOG.rst new file mode 100644 index 000000000..dafdc2ab2 --- /dev/null +++ b/rosidl_generator_type_hash/CHANGELOG.rst @@ -0,0 +1,141 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package rosidl_generator_c +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +3.3.1 (2022-11-02) +------------------ + +3.3.0 (2022-09-08) +------------------ +* Move rosidl_generator_c/cpp tests to a separate package (`#701 `_) +* Move rosidl_cmake Python module to a new package rosidl_pycommon (`#696 `_) + Deprecate the Python module in rosidl_cmake and move the implementation to the new package rosidl_pycommon. +* Add namespaced ALIAS target to easily consume generated libraries via add_subdirectory (`#605 `_) +* Contributors: Jacob Perron, Silvio Traversaro + +3.2.1 (2022-06-21) +------------------ + +3.2.0 (2022-05-04) +------------------ + +3.1.3 (2022-04-08) +------------------ + +3.1.2 (2022-04-05) +------------------ +* Fix error handling when copying C sequence messages (`#671 `_) +* Contributors: Michel Hidalgo + +3.1.1 (2022-03-28) +------------------ +* Install generated headers to include/${PROJECT_NAME} (`#670 `_) +* Misc cleanup in the rosidl generator extensions (`#662 `_) +* Set the output size unconditionally when copying sequences (`#669 `_) +* Contributors: Nikolai Morin, Shane Loretz + +3.1.0 (2022-03-01) +------------------ +* Implement copy function for C messages (`#650 `_) +* Implement equality operator function for C messages. (`#648 `_) +* Generate documentation in generated C header files based on ROS interfaces comments (`#593 `_) +* Contributors: Ivan Santiago Paunovic, Michel Hidalgo + +3.0.1 (2022-01-13) +------------------ + +3.0.0 (2021-11-05) +------------------ +* Update package maintainers (`#624 `_) +* Make rosidl packages use FindPython3 instead of FindPythonInterp (`#612 `_) +* Contributors: Michel Hidalgo, Shane Loretz + +2.5.0 (2021-08-10) +------------------ +* Revert "Bundle and ensure the exportation of rosidl generated targets" (`#611 `_) +* Contributors: Michel Hidalgo + +2.4.0 (2021-07-12) +------------------ +* Bundle and ensure the exportation of rosidl generated targets (`#601 `_) +* Contributors: Michel Hidalgo + +2.3.0 (2021-06-11) +------------------ +* Fix a cpplint allocator regression. (`#590 `_) +* Use RCUtils allocators in rosidl_generator_c (`#584 `_) +* Contributors: Chris Lalancette, Pablo Garrido + +2.2.1 (2021-04-06) +------------------ + +2.2.0 (2021-03-18) +------------------ +* Expose C code generation via rosidl generate CLI (`#569 `_) +* Contributors: Michel Hidalgo + +2.1.0 (2021-03-09) +------------------ + +2.0.3 (2021-01-25) +------------------ + +2.0.2 (2020-12-08) +------------------ +* Strip action service suffixes from C include prefix (`#538 `_) +* Update the maintainers of this repository. (`#536 `_) +* Contributors: Chris Lalancette, Jacob Perron + +2.0.1 (2020-09-28) +------------------ + +2.0.0 (2020-09-24) +------------------ +* Fix the declared language for a few packages (`#530 `_) +* Contributors: Scott K Logan + +1.1.0 (2020-08-17) +------------------ +* Do not depend on rosidl_runtime_c when tests are disabled (`#503 `_) +* Contributors: Ben Wolsieffer + +1.0.1 (2020-06-03) +------------------ + +1.0.0 (2020-05-22) +------------------ + +0.9.2 (2020-05-19) +------------------ + +0.9.1 (2020-05-08) +------------------ + +0.9.0 (2020-04-24) +------------------ +* Export targets in addition to include directories / libraries (`#473 `_) +* Move non-entry point headers into detail subdirectory (`#461 `_) +* Rename rosidl_generator_c 'namespace' to rosidl_runtime_c (`#458 `_) +* Only export ament_cmake_core instead of ament_cmake (`#459 `_) +* Split rosidl_generator_c and rosidl_generator_cpp in two: rosidl_generator_x and rosidl_runtime_x (`#442 `_) +* Added rosidl_generator_c as a member of group rosidl_runtime_packages (`#440 `_) +* Style update to match uncrustify with explicit language (`#439 `_) +* Code style only: wrap after open parenthesis if not in one line (`#435 `_) +* Use f-string (`#436 `_) +* Move repeated logic for C include prefix into common function (`#432 `_) +* Contributors: Alejandro Hernández Cordero, Dirk Thomas, Jacob Perron + +0.8.2 (2020-01-17) +------------------ +* Fix double free issue when initialization is failed (`#423 `_) +* Contributors: DongheeYe + +0.8.1 (2019-10-23) +------------------ + +0.8.0 (2019-09-24) +------------------ +* [rosidl_generator_c] Updated tests for new msg types from test_interface_files (`#398 `_) +* use latin-1 encoding when reading/writing .idl files, prepend BOM to generated C/C++ files when necessary (`#391 `_) +* Set _FOUND to trick ament_target_dependencies() for test (`#396 `_) +* Contributors: Dirk Thomas, Shane Loretz, Siddharth Kucheria diff --git a/rosidl_generator_type_hash/CMakeLists.txt b/rosidl_generator_type_hash/CMakeLists.txt new file mode 100644 index 000000000..0503e9b20 --- /dev/null +++ b/rosidl_generator_type_hash/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.12) + +project(rosidl_generator_type_hash) + +find_package(ament_cmake_python REQUIRED) +find_package(ament_cmake_ros REQUIRED) + +ament_index_register_resource("rosidl_generator_packages") +ament_python_install_package(${PROJECT_NAME}) + +if(BUILD_TESTING) + find_package(ament_lint_auto REQUIRED) + ament_lint_auto_find_test_dependencies() +endif() + +ament_package() + +install( + PROGRAMS bin/rosidl_generator_c + DESTINATION lib/rosidl_generator_c +) +install( + DIRECTORY cmake resource + DESTINATION share/${PROJECT_NAME} +) diff --git a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash new file mode 100755 index 000000000..7d863bf7b --- /dev/null +++ b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys + +try: + from rosidl_generator_c import generate_c +except ImportError: + # modifying sys.path and importing the Python package with the same + # name as this script does not work on Windows + rosidl_generator_c_root = os.path.dirname(os.path.dirname(__file__)) + rosidl_generator_c_module = os.path.join( + rosidl_generator_c_root, 'rosidl_generator_c', '__init__.py') + if not os.path.exists(rosidl_generator_c_module): + raise + from importlib.machinery import SourceFileLoader + + loader = SourceFileLoader('rosidl_generator_c', rosidl_generator_c_module) + rosidl_generator_c = loader.load_module() + generate_c = rosidl_generator_c.generate_c + + +def main(argv=sys.argv[1:]): + parser = argparse.ArgumentParser( + description='Generate hashable representations and hashes of ROS interfaces.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '--generator-arguments-file', + required=True, + help='The location of the file containing the generator arguments') + args = parser.parse_args(argv) + + generate_c(args.generator_arguments_file) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake new file mode 100644 index 000000000..b024fe054 --- /dev/null +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -0,0 +1,200 @@ +# Copyright 2015-2018 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +find_package(rcutils REQUIRED) +find_package(rosidl_runtime_c REQUIRED) +find_package(rosidl_typesupport_interface REQUIRED) + +set(rosidl_generate_interfaces_c_IDL_TUPLES + ${rosidl_generate_interfaces_IDL_TUPLES}) +set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_c/${PROJECT_NAME}") +set(_generated_headers "") +set(_generated_sources "") +foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) + get_filename_component(_parent_folder "${_abs_idl_file}" DIRECTORY) + get_filename_component(_parent_folder "${_parent_folder}" NAME) + get_filename_component(_idl_name "${_abs_idl_file}" NAME_WE) + string_camel_case_to_lower_case_underscore("${_idl_name}" _header_name) + list(APPEND _generated_headers + "${_output_path}/${_parent_folder}/${_header_name}.h" + "${_output_path}/${_parent_folder}/detail/${_header_name}__functions.h" + "${_output_path}/${_parent_folder}/detail/${_header_name}__struct.h" + "${_output_path}/${_parent_folder}/detail/${_header_name}__type_support.h" + ) + list(APPEND _generated_sources + "${_output_path}/${_parent_folder}/detail/${_header_name}__functions.c" + ) +endforeach() + +set(_dependency_files "") +set(_dependencies "") +foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) + foreach(_idl_file ${${_pkg_name}_IDL_FILES}) + set(_abs_idl_file "${${_pkg_name}_DIR}/../${_idl_file}") + normalize_path(_abs_idl_file "${_abs_idl_file}") + list(APPEND _dependency_files "${_abs_idl_file}") + list(APPEND _dependencies "${_pkg_name}:${_abs_idl_file}") + endforeach() +endforeach() + +set(target_dependencies + "${rosidl_generator_c_BIN}" + ${rosidl_generator_c_GENERATOR_FILES} + "${rosidl_generator_c_TEMPLATE_DIR}/action__type_support.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/idl.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/idl__functions.c.em" + "${rosidl_generator_c_TEMPLATE_DIR}/idl__functions.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/idl__struct.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/idl__type_support.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/msg__functions.c.em" + "${rosidl_generator_c_TEMPLATE_DIR}/msg__functions.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/msg__struct.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/msg__type_support.h.em" + "${rosidl_generator_c_TEMPLATE_DIR}/srv__type_support.h.em" + ${rosidl_generate_interfaces_ABS_IDL_FILES} + ${_dependency_files}) +foreach(dep ${target_dependencies}) + if(NOT EXISTS "${dep}") + message(FATAL_ERROR "Target dependency '${dep}' does not exist") + endif() +endforeach() + +set(generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_c__arguments.json") +rosidl_write_generator_arguments( + "${generator_arguments_file}" + PACKAGE_NAME "${PROJECT_NAME}" + IDL_TUPLES "${rosidl_generate_interfaces_c_IDL_TUPLES}" + ROS_INTERFACE_DEPENDENCIES "${_dependencies}" + OUTPUT_DIR "${_output_path}" + TEMPLATE_DIR "${rosidl_generator_c_TEMPLATE_DIR}" + TARGET_DEPENDENCIES ${target_dependencies} +) + +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +add_custom_command( + OUTPUT ${_generated_headers} ${_generated_sources} + COMMAND Python3::Interpreter + ARGS ${rosidl_generator_c_BIN} + --generator-arguments-file "${generator_arguments_file}" + DEPENDS ${target_dependencies} + COMMENT "Generating C code for ROS interfaces" + VERBATIM +) + +# generate header to switch between export and import for a specific package +set(_visibility_control_file + "${_output_path}/msg/rosidl_generator_c__visibility_control.h") +string(TOUPPER "${PROJECT_NAME}" PROJECT_NAME_UPPER) +configure_file( + "${rosidl_generator_c_TEMPLATE_DIR}/rosidl_generator_c__visibility_control.h.in" + "${_visibility_control_file}" + @ONLY +) + +list(APPEND _generated_msg_headers "${_visibility_control_file}") + +set(_target_suffix "__rosidl_generator_c") + +add_library(${rosidl_generate_interfaces_TARGET}${_target_suffix} ${rosidl_generator_c_LIBRARY_TYPE} + ${_generated_headers} ${_generated_sources}) +add_library(${PROJECT_NAME}::${rosidl_generate_interfaces_TARGET}${_target_suffix} ALIAS + ${rosidl_generate_interfaces_TARGET}${_target_suffix}) +if(rosidl_generate_interfaces_LIBRARY_NAME) + set_target_properties(${rosidl_generate_interfaces_TARGET}${_target_suffix} + PROPERTIES OUTPUT_NAME "${rosidl_generate_interfaces_LIBRARY_NAME}${_target_suffix}") +endif() +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set_target_properties(${rosidl_generate_interfaces_TARGET}${_target_suffix} PROPERTIES + C_STANDARD 11 + COMPILE_OPTIONS -Wall -Wextra -Wpedantic) +endif() +set_property(TARGET ${rosidl_generate_interfaces_TARGET}${_target_suffix} + PROPERTY DEFINE_SYMBOL "ROSIDL_GENERATOR_C_BUILDING_DLL_${PROJECT_NAME}") +target_include_directories(${rosidl_generate_interfaces_TARGET}${_target_suffix} + PUBLIC + "$" + "$" +) +foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) + # Depend on targets generated by this generator in dependency packages + target_link_libraries( + ${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBLIC + ${${_pkg_name}_TARGETS${_target_suffix}}) +endforeach() + +target_link_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBLIC + rosidl_runtime_c::rosidl_runtime_c + rosidl_typesupport_interface::rosidl_typesupport_interface + rcutils::rcutils) + +# Make top level generation target depend on this generated library +add_dependencies( + ${rosidl_generate_interfaces_TARGET} + ${rosidl_generate_interfaces_TARGET}${_target_suffix} +) + +if(NOT rosidl_generate_interfaces_SKIP_INSTALL) + install( + DIRECTORY ${_output_path}/ + DESTINATION "include/${PROJECT_NAME}/${PROJECT_NAME}" + PATTERN "*.h" + ) + + # Export old-style CMake variables + ament_export_include_directories("include/${PROJECT_NAME}") + ament_export_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix}) + + # Export modern CMake targets + ament_export_targets(export_${rosidl_generate_interfaces_TARGET}${_target_suffix}) + rosidl_export_typesupport_targets(${_target_suffix} + ${rosidl_generate_interfaces_TARGET}${_target_suffix}) + + install( + TARGETS ${rosidl_generate_interfaces_TARGET}${_target_suffix} + EXPORT export_${rosidl_generate_interfaces_TARGET}${_target_suffix} + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin + ) + + ament_export_dependencies( + "rosidl_runtime_c" + "rosidl_typesupport_interface" + "rcutils") +endif() + +if(BUILD_TESTING AND rosidl_generate_interfaces_ADD_LINTER_TESTS) + find_package(ament_cmake_cppcheck REQUIRED) + ament_cppcheck( + TESTNAME "cppcheck_rosidl_generated_c" + "${_output_path}") + + find_package(ament_cmake_cpplint REQUIRED) + get_filename_component(_cpplint_root "${_output_path}" DIRECTORY) + ament_cpplint( + TESTNAME "cpplint_rosidl_generated_c" + # the generated code might contain longer lines for templated types + MAX_LINE_LENGTH 999 + ROOT "${_cpplint_root}" + "${_output_path}") + + find_package(ament_cmake_uncrustify REQUIRED) + ament_uncrustify( + TESTNAME "uncrustify_rosidl_generated_c" + # the generated code might contain longer lines for templated types + # a value of zero tells uncrustify to ignore line length + MAX_LINE_LENGTH 0 + "${_output_path}") +endif() diff --git a/rosidl_generator_type_hash/package.xml b/rosidl_generator_type_hash/package.xml new file mode 100644 index 000000000..adb0dffcf --- /dev/null +++ b/rosidl_generator_type_hash/package.xml @@ -0,0 +1,34 @@ + + + + rosidl_generator_type_hash + 3.3.1 + Generate the ROS interfaces in C. + + Emerson Knapp + + Apache License 2.0 + + Emerson Knapp + + ament_cmake_python + ament_cmake_ros + + ament_cmake_core + python3 + rosidl_pycommon + + ament_index_python + rosidl_cli + rosidl_parser + rcutils + + ament_lint_auto + ament_lint_common + + rosidl_generator_packages + + + ament_cmake + + diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py new file mode 100644 index 000000000..02aa5e884 --- /dev/null +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -0,0 +1,220 @@ +# Copyright 2015 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rosidl_parser.definition import AbstractGenericString +from rosidl_parser.definition import AbstractSequence +from rosidl_parser.definition import AbstractString +from rosidl_parser.definition import AbstractType +from rosidl_parser.definition import AbstractWString +from rosidl_parser.definition import Array +from rosidl_parser.definition import BasicType +from rosidl_parser.definition import CHARACTER_TYPES +from rosidl_parser.definition import NamespacedType +from rosidl_parser.definition import OCTET_TYPE +from rosidl_pycommon import convert_camel_case_to_lower_case_underscore +from rosidl_pycommon import generate_files + + +def generate_c(generator_arguments_file): + mapping = { + 'idl.h.em': '%s.h', + 'idl__functions.c.em': 'detail/%s__functions.c', + 'idl__functions.h.em': 'detail/%s__functions.h', + 'idl__struct.h.em': 'detail/%s__struct.h', + 'idl__type_support.h.em': 'detail/%s__type_support.h', + } + return generate_files( + generator_arguments_file, mapping, + post_process_callback=prefix_with_bom_if_necessary) + + +def prefix_with_bom_if_necessary(content): + try: + content.encode('ASCII') + except UnicodeError: + prefix = '\ufeff' + \ + '// NOLINT: This file starts with a BOM ' + \ + 'since it contain non-ASCII characters\n' + content = prefix + content + return content + + +BASIC_IDL_TYPES_TO_C = { + 'float': 'float', + 'double': 'double', + 'long double': 'long double', + 'char': 'signed char', + 'wchar': 'uint16_t', + 'boolean': 'bool', + 'octet': 'uint8_t', + 'uint8': 'uint8_t', + 'int8': 'int8_t', + 'uint16': 'uint16_t', + 'int16': 'int16_t', + 'uint32': 'uint32_t', + 'int32': 'int32_t', + 'uint64': 'uint64_t', + 'int64': 'int64_t', +} + + +def idl_structure_type_to_c_include_prefix(namespaced_type, subdirectory=None): + parts = [ + convert_camel_case_to_lower_case_underscore(x) + for x in (namespaced_type.namespaced_name())] + if subdirectory is not None: + parts[-1:-1] = [subdirectory] + include_prefix = '/'.join(parts) + # Strip service or action suffixes + if include_prefix.endswith('__request'): + include_prefix = include_prefix[:-9] + elif include_prefix.endswith('__response'): + include_prefix = include_prefix[:-10] + if include_prefix.endswith('__goal'): + include_prefix = include_prefix[:-6] + elif include_prefix.endswith('__result'): + include_prefix = include_prefix[:-8] + elif include_prefix.endswith('__feedback'): + include_prefix = include_prefix[:-10] + elif include_prefix.endswith('__send_goal'): + include_prefix = include_prefix[:-11] + elif include_prefix.endswith('__get_result'): + include_prefix = include_prefix[:-12] + return include_prefix + + +def idl_structure_type_to_c_typename(namespaced_type): + return '__'.join(namespaced_type.namespaced_name()) + + +def idl_structure_type_sequence_to_c_typename(namespaced_type): + return idl_structure_type_to_c_typename(namespaced_type) + '__Sequence' + + +def interface_path_to_string(interface_path): + return '/'.join( + list(interface_path.parents[0].parts) + [interface_path.stem]) + + +def idl_declaration_to_c(type_, name): + """ + Convert an IDL type into the C declaration. + + Example input: uint32, std_msgs/String + Example output: uint32_t, char * + + @param type_: The message type + @type type_: rosidl_parser.Type + @param type_: The member name + @type type_: str + """ + if isinstance(type_, AbstractGenericString): + return basetype_to_c(type_) + ' ' + name + if isinstance(type_, Array): + return basetype_to_c(type_.value_type) + ' ' + name + '[' + str(type_.size) + ']' + return idl_type_to_c(type_) + ' ' + name + + +def idl_type_to_c(type_): + if isinstance(type_, Array): + assert False, 'The array size is part of the variable' + if isinstance(type_, AbstractSequence): + if isinstance(type_.value_type, BasicType): + c_type = 'rosidl_runtime_c__' + type_.value_type.typename.replace(' ', '_') + else: + c_type = basetype_to_c(type_.value_type) + c_type += '__Sequence' + return c_type + return basetype_to_c(type_) + + +def basetype_to_c(basetype): + if isinstance(basetype, BasicType): + return BASIC_IDL_TYPES_TO_C[basetype.typename] + if isinstance(basetype, AbstractString): + return 'rosidl_runtime_c__String' + if isinstance(basetype, AbstractWString): + return 'rosidl_runtime_c__U16String' + if isinstance(basetype, NamespacedType): + return idl_structure_type_to_c_typename(basetype) + assert False, str(basetype) + + +def value_to_c(type_, value): + assert isinstance(type_, AbstractType) + assert value is not None + + if isinstance(type_, AbstractString): + return '"%s"' % escape_string(value) + + if isinstance(type_, AbstractWString): + return 'u"%s"' % escape_wstring(value) + + return basic_value_to_c(type_, value) + + +def basic_value_to_c(type_, value): + assert isinstance(type_, BasicType) + assert value is not None + + if 'boolean' == type_.typename: + return 'true' if value else 'false' + + if type_.typename in ( + *CHARACTER_TYPES, + OCTET_TYPE, + 'int8', + 'uint8', + 'int16', + 'uint16', + ): + return str(value) + + if type_.typename == 'int32': + # Handle edge case for INT32_MIN + # Specifically, MSVC is not happy in this case + if -2147483648 == value: + return '({0}l - 1)'.format(value + 1) + return f'{value}l' + + if type_.typename == 'uint32': + return f'{value}ul' + + if type_.typename == 'int64': + # Handle edge case for INT64_MIN + # See https://en.cppreference.com/w/cpp/language/integer_literal + if -9223372036854775808 == value: + return '({0}ll - 1)'.format(value + 1) + return f'{value}ll' + + if type_.typename == 'uint64': + return f'{value}ull' + + if 'float' == type_.typename: + return f'{value}f' + + if 'double' == type_.typename: + return f'{value}l' + + assert False, "unknown basic type '%s'" % type_ + + +def escape_string(s): + s = s.replace('\\', '\\\\') + s = s.replace('"', r'\"') + return s + + +def escape_wstring(s): + return escape_string(s) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py new file mode 100644 index 000000000..f6c786d7d --- /dev/null +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py @@ -0,0 +1,81 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib + +from ament_index_python import get_package_share_directory +from rosidl_cli.command.generate.extensions import GenerateCommandExtension +from rosidl_cli.command.helpers import generate_visibility_control_file +from rosidl_cli.command.helpers import legacy_generator_arguments_file +from rosidl_cli.command.translate.api import translate + +from rosidl_generator_c import generate_c + + +class GenerateC(GenerateCommandExtension): + + def generate( + self, + package_name, + interface_files, + include_paths, + output_path + ): + generated_files = [] + + package_share_path = \ + pathlib.Path(get_package_share_directory('rosidl_generator_c')) + templates_path = package_share_path / 'resource' + + # Normalize interface definition format to .idl + idl_interface_files = [] + non_idl_interface_files = [] + for path in interface_files: + if not path.endswith('.idl'): + non_idl_interface_files.append(path) + else: + idl_interface_files.append(path) + if non_idl_interface_files: + idl_interface_files.extend(translate( + package_name=package_name, + interface_files=non_idl_interface_files, + include_paths=include_paths, + output_format='idl', + output_path=output_path / 'tmp', + )) + + # Generate visibility control file + visibility_control_file_template_path = \ + templates_path / 'rosidl_generator_c__visibility_control.h.in' + visibility_control_file_path = \ + output_path / 'msg' / 'rosidl_generator_c__visibility_control.h' + + generate_visibility_control_file( + package_name=package_name, + template_path=visibility_control_file_template_path, + output_path=visibility_control_file_path + ) + generated_files.append(visibility_control_file_path) + + # Generate code + with legacy_generator_arguments_file( + package_name=package_name, + interface_files=idl_interface_files, + include_paths=include_paths, + templates_path=templates_path, + output_path=output_path + ) as path_to_arguments_file: + generated_files.extend(generate_c(path_to_arguments_file)) + + return generated_files diff --git a/rosidl_generator_type_hash/setup.cfg b/rosidl_generator_type_hash/setup.cfg new file mode 100644 index 000000000..a269e3416 --- /dev/null +++ b/rosidl_generator_type_hash/setup.cfg @@ -0,0 +1,3 @@ +[options.entry_points] +rosidl_cli.command.generate.type_extensions = + c = rosidl_generator_type_hash.cli:GenerateTypeHash From 50e6c5dbfcd119464912417eaba4099cbf91b681 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 2 Feb 2023 17:43:27 -0800 Subject: [PATCH 09/65] WIP refactoring as a new rosidl generator Signed-off-by: Emerson Knapp --- rosidl_generator_c/package.xml | 1 + rosidl_generator_type_hash/CMakeLists.txt | 10 +- .../bin/rosidl_generator_type_hash | 18 +- .../cmake/register_type_hash.cmake | 16 + ...erator_type_hash_generate_interfaces.cmake | 153 ++----- .../rosidl_generator_type_hash-extras.cmake | 6 + ...rosidl_generator_type_hash-extras.cmake.in | 6 + .../rosidl_generator_type_hash/__init__.py | 421 ++++++++++-------- rosidl_parser/rosidl_parser/type_hash.py | 161 ------- rosidl_pycommon/rosidl_pycommon/__init__.py | 14 +- 10 files changed, 309 insertions(+), 497 deletions(-) create mode 100644 rosidl_generator_type_hash/cmake/register_type_hash.cmake create mode 100644 rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake create mode 100644 rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in delete mode 100644 rosidl_parser/rosidl_parser/type_hash.py diff --git a/rosidl_generator_c/package.xml b/rosidl_generator_c/package.xml index 63a94991a..f76ef4c49 100644 --- a/rosidl_generator_c/package.xml +++ b/rosidl_generator_c/package.xml @@ -24,6 +24,7 @@ python3 rosidl_pycommon + rosidl_generator_type_hash rosidl_typesupport_interface rcutils diff --git a/rosidl_generator_type_hash/CMakeLists.txt b/rosidl_generator_type_hash/CMakeLists.txt index 0503e9b20..432d3cddd 100644 --- a/rosidl_generator_type_hash/CMakeLists.txt +++ b/rosidl_generator_type_hash/CMakeLists.txt @@ -13,13 +13,15 @@ if(BUILD_TESTING) ament_lint_auto_find_test_dependencies() endif() -ament_package() +ament_package( + CONFIG_EXTRAS "${PROJECT_NAME}-extras.cmake.in" +) install( - PROGRAMS bin/rosidl_generator_c - DESTINATION lib/rosidl_generator_c + PROGRAMS bin/${PROJECT_NAME} + DESTINATION lib/${PROJECT_NAME} ) install( - DIRECTORY cmake resource + DIRECTORY cmake # resource DESTINATION share/${PROJECT_NAME} ) diff --git a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash index 7d863bf7b..277f99a7f 100755 --- a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash +++ b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash @@ -5,20 +5,20 @@ import os import sys try: - from rosidl_generator_c import generate_c + from rosidl_generator_type_hash import generate_type_hash except ImportError: # modifying sys.path and importing the Python package with the same # name as this script does not work on Windows - rosidl_generator_c_root = os.path.dirname(os.path.dirname(__file__)) - rosidl_generator_c_module = os.path.join( - rosidl_generator_c_root, 'rosidl_generator_c', '__init__.py') - if not os.path.exists(rosidl_generator_c_module): + rosidl_generator_type_hash_root = os.path.dirname(os.path.dirname(__file__)) + rosidl_generator_type_hash_module = os.path.join( + rosidl_generator_type_hash_root, 'rosidl_generator_type_hash', '__init__.py') + if not os.path.exists(rosidl_generator_type_hash_module): raise from importlib.machinery import SourceFileLoader - loader = SourceFileLoader('rosidl_generator_c', rosidl_generator_c_module) - rosidl_generator_c = loader.load_module() - generate_c = rosidl_generator_c.generate_c + loader = SourceFileLoader('rosidl_generator_type_hash', rosidl_generator_type_hash_module) + rosidl_generator_type_hash = loader.load_module() + generate_type_hash = rosidl_generator_type_hash.generate_type_hash def main(argv=sys.argv[1:]): @@ -31,7 +31,7 @@ def main(argv=sys.argv[1:]): help='The location of the file containing the generator arguments') args = parser.parse_args(argv) - generate_c(args.generator_arguments_file) + generate_type_hash(args.generator_arguments_file) if __name__ == '__main__': diff --git a/rosidl_generator_type_hash/cmake/register_type_hash.cmake b/rosidl_generator_type_hash/cmake/register_type_hash.cmake new file mode 100644 index 000000000..d448d1df5 --- /dev/null +++ b/rosidl_generator_type_hash/cmake/register_type_hash.cmake @@ -0,0 +1,16 @@ +macro(rosidl_generator_type_hash_extras BIN GENERATOR_FILES TEMPLATE_DIR) + find_package(ament_cmake_core QUIET REQUIRED) + ament_register_extension( + "rosidl_generate_idl_interfaces" + "rosidl_generator_type_hash" + "rosidl_generator_type_hash_generate_interfaces.cmake") + + normalize_path(BIN "${BIN}") + set(rosidl_generator_type_hash_BIN "${BIN}") + + normalize_path(GENERATOR_FILES "${GENERATOR_FILES}") + set(rosidl_generator_type_hash_GENERATOR_FILES "${GENERATOR_FILES}") + + normalize_path(TEMPLATE_DIR "${TEMPLATE_DIR}") + set(rosidl_generator_type_hash_TEMPLATE_DIR "${TEMPLATE_DIR}") +endmacro() diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index b024fe054..42027e822 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -12,28 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -find_package(rcutils REQUIRED) -find_package(rosidl_runtime_c REQUIRED) -find_package(rosidl_typesupport_interface REQUIRED) - -set(rosidl_generate_interfaces_c_IDL_TUPLES +set(rosidl_generate_interfaces_type_hash_IDL_TUPLES ${rosidl_generate_interfaces_IDL_TUPLES}) -set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_c/${PROJECT_NAME}") -set(_generated_headers "") -set(_generated_sources "") +set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash/${PROJECT_NAME}") +set(_generated_jsons "") +set(_generated_hashes "") foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) get_filename_component(_parent_folder "${_abs_idl_file}" DIRECTORY) get_filename_component(_parent_folder "${_parent_folder}" NAME) get_filename_component(_idl_name "${_abs_idl_file}" NAME_WE) - string_camel_case_to_lower_case_underscore("${_idl_name}" _header_name) - list(APPEND _generated_headers - "${_output_path}/${_parent_folder}/${_header_name}.h" - "${_output_path}/${_parent_folder}/detail/${_header_name}__functions.h" - "${_output_path}/${_parent_folder}/detail/${_header_name}__struct.h" - "${_output_path}/${_parent_folder}/detail/${_header_name}__type_support.h" + string_camel_case_to_lower_case_underscore("${_idl_name}" _idl_stem) + list(APPEND _generated_files + "${_output_path}/${_parent_folder}/${_idl_stem}.json" ) - list(APPEND _generated_sources - "${_output_path}/${_parent_folder}/detail/${_header_name}__functions.c" + list(APPEND _generated_hashes + "${_output_path}/${_parent_folder}/${_idl_stem}.sha256" ) endforeach() @@ -49,19 +42,8 @@ foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) endforeach() set(target_dependencies - "${rosidl_generator_c_BIN}" - ${rosidl_generator_c_GENERATOR_FILES} - "${rosidl_generator_c_TEMPLATE_DIR}/action__type_support.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/idl.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/idl__functions.c.em" - "${rosidl_generator_c_TEMPLATE_DIR}/idl__functions.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/idl__struct.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/idl__type_support.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/msg__functions.c.em" - "${rosidl_generator_c_TEMPLATE_DIR}/msg__functions.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/msg__struct.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/msg__type_support.h.em" - "${rosidl_generator_c_TEMPLATE_DIR}/srv__type_support.h.em" + "${rosidl_generator_type_hash_BIN}" + ${rosidl_generator_type_hash_GENERATOR_FILES} ${rosidl_generate_interfaces_ABS_IDL_FILES} ${_dependency_files}) foreach(dep ${target_dependencies}) @@ -70,76 +52,48 @@ foreach(dep ${target_dependencies}) endif() endforeach() -set(generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_c__arguments.json") +set(generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash__arguments.json") rosidl_write_generator_arguments( "${generator_arguments_file}" PACKAGE_NAME "${PROJECT_NAME}" - IDL_TUPLES "${rosidl_generate_interfaces_c_IDL_TUPLES}" + IDL_TUPLES "${rosidl_generate_interfaces_type_hash_IDL_TUPLES}" ROS_INTERFACE_DEPENDENCIES "${_dependencies}" OUTPUT_DIR "${_output_path}" - TEMPLATE_DIR "${rosidl_generator_c_TEMPLATE_DIR}" + TEMPLATE_DIR "${rosidl_generator_type_hash_TEMPLATE_DIR}" TARGET_DEPENDENCIES ${target_dependencies} ) find_package(Python3 REQUIRED COMPONENTS Interpreter) add_custom_command( - OUTPUT ${_generated_headers} ${_generated_sources} + OUTPUT ${_generated_jsons} ${_generated_hashes} COMMAND Python3::Interpreter - ARGS ${rosidl_generator_c_BIN} + ARGS ${rosidl_generator_type_hash_BIN} --generator-arguments-file "${generator_arguments_file}" DEPENDS ${target_dependencies} - COMMENT "Generating C code for ROS interfaces" + COMMENT "Generating type hashes for ROS interfaces" VERBATIM ) -# generate header to switch between export and import for a specific package -set(_visibility_control_file - "${_output_path}/msg/rosidl_generator_c__visibility_control.h") -string(TOUPPER "${PROJECT_NAME}" PROJECT_NAME_UPPER) -configure_file( - "${rosidl_generator_c_TEMPLATE_DIR}/rosidl_generator_c__visibility_control.h.in" - "${_visibility_control_file}" - @ONLY -) - -list(APPEND _generated_msg_headers "${_visibility_control_file}") +set(_target_suffix "__rosidl_generator_type_hash") -set(_target_suffix "__rosidl_generator_c") - -add_library(${rosidl_generate_interfaces_TARGET}${_target_suffix} ${rosidl_generator_c_LIBRARY_TYPE} - ${_generated_headers} ${_generated_sources}) -add_library(${PROJECT_NAME}::${rosidl_generate_interfaces_TARGET}${_target_suffix} ALIAS - ${rosidl_generate_interfaces_TARGET}${_target_suffix}) +add_custom_target( + ${rosidl_generate_interfaces_TARGET}${_target_suffix} + DEPENDS ${_generated_jsons} ${_generated_hashes}) +# add_custom_target(${PROJECT_NAME}::${rosidl_generate_interfaces_TARGET}${_target_suffix} ALIAS +# ${rosidl_generate_interfaces_TARGET}${_target_suffix}) if(rosidl_generate_interfaces_LIBRARY_NAME) set_target_properties(${rosidl_generate_interfaces_TARGET}${_target_suffix} PROPERTIES OUTPUT_NAME "${rosidl_generate_interfaces_LIBRARY_NAME}${_target_suffix}") endif() -if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set_target_properties(${rosidl_generate_interfaces_TARGET}${_target_suffix} PROPERTIES - C_STANDARD 11 - COMPILE_OPTIONS -Wall -Wextra -Wpedantic) -endif() -set_property(TARGET ${rosidl_generate_interfaces_TARGET}${_target_suffix} - PROPERTY DEFINE_SYMBOL "ROSIDL_GENERATOR_C_BUILDING_DLL_${PROJECT_NAME}") -target_include_directories(${rosidl_generate_interfaces_TARGET}${_target_suffix} - PUBLIC - "$" - "$" -) -foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) - # Depend on targets generated by this generator in dependency packages - target_link_libraries( - ${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBLIC - ${${_pkg_name}_TARGETS${_target_suffix}}) -endforeach() - -target_link_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBLIC - rosidl_runtime_c::rosidl_runtime_c - rosidl_typesupport_interface::rosidl_typesupport_interface - rcutils::rcutils) - -# Make top level generation target depend on this generated library +# foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) +# # Depend on targets generated by this generator in dependency packages +# target_link_libraries( +# ${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBLIC +# ${${_pkg_name}_TARGETS${_target_suffix}}) +# endforeach() + +# # Make top level generation target depend on this generated library add_dependencies( ${rosidl_generate_interfaces_TARGET} ${rosidl_generate_interfaces_TARGET}${_target_suffix} @@ -149,7 +103,7 @@ if(NOT rosidl_generate_interfaces_SKIP_INSTALL) install( DIRECTORY ${_output_path}/ DESTINATION "include/${PROJECT_NAME}/${PROJECT_NAME}" - PATTERN "*.h" + PATTERN "*.json" ) # Export old-style CMake variables @@ -161,40 +115,11 @@ if(NOT rosidl_generate_interfaces_SKIP_INSTALL) rosidl_export_typesupport_targets(${_target_suffix} ${rosidl_generate_interfaces_TARGET}${_target_suffix}) - install( - TARGETS ${rosidl_generate_interfaces_TARGET}${_target_suffix} - EXPORT export_${rosidl_generate_interfaces_TARGET}${_target_suffix} - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION bin - ) - - ament_export_dependencies( - "rosidl_runtime_c" - "rosidl_typesupport_interface" - "rcutils") -endif() - -if(BUILD_TESTING AND rosidl_generate_interfaces_ADD_LINTER_TESTS) - find_package(ament_cmake_cppcheck REQUIRED) - ament_cppcheck( - TESTNAME "cppcheck_rosidl_generated_c" - "${_output_path}") - - find_package(ament_cmake_cpplint REQUIRED) - get_filename_component(_cpplint_root "${_output_path}" DIRECTORY) - ament_cpplint( - TESTNAME "cpplint_rosidl_generated_c" - # the generated code might contain longer lines for templated types - MAX_LINE_LENGTH 999 - ROOT "${_cpplint_root}" - "${_output_path}") - - find_package(ament_cmake_uncrustify REQUIRED) - ament_uncrustify( - TESTNAME "uncrustify_rosidl_generated_c" - # the generated code might contain longer lines for templated types - # a value of zero tells uncrustify to ignore line length - MAX_LINE_LENGTH 0 - "${_output_path}") + # install( + # TARGETS ${rosidl_generate_interfaces_TARGET}${_target_suffix} + # EXPORT export_${rosidl_generate_interfaces_TARGET}${_target_suffix} + # ARCHIVE DESTINATION lib + # LIBRARY DESTINATION lib + # RUNTIME DESTINATION bin + # ) endif() diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake new file mode 100644 index 000000000..8a18b8053 --- /dev/null +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake @@ -0,0 +1,6 @@ +include("${CMAKE_CURRENT_LIST_DIR}/register_type_hash.cmake") +rosidl_generator_type_hash_extras( + "${rosidl_generator_type_hash_DIR}/../../../lib/rosidl_generator_type_hash/rosidl_generator_type_hash" + "${rosidl_generator_type_hash_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_hash/__init__.py" + "${rosidl_generator_type_hash_DIR}/../resource" +) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in new file mode 100644 index 000000000..8a18b8053 --- /dev/null +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in @@ -0,0 +1,6 @@ +include("${CMAKE_CURRENT_LIST_DIR}/register_type_hash.cmake") +rosidl_generator_type_hash_extras( + "${rosidl_generator_type_hash_DIR}/../../../lib/rosidl_generator_type_hash/rosidl_generator_type_hash" + "${rosidl_generator_type_hash_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_hash/__init__.py" + "${rosidl_generator_type_hash_DIR}/../resource" +) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 02aa5e884..fabab24e3 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -12,209 +12,238 @@ # See the License for the specific language governing permissions and # limitations under the License. -from rosidl_parser.definition import AbstractGenericString -from rosidl_parser.definition import AbstractSequence -from rosidl_parser.definition import AbstractString -from rosidl_parser.definition import AbstractType -from rosidl_parser.definition import AbstractWString -from rosidl_parser.definition import Array -from rosidl_parser.definition import BasicType -from rosidl_parser.definition import CHARACTER_TYPES -from rosidl_parser.definition import NamespacedType -from rosidl_parser.definition import OCTET_TYPE -from rosidl_pycommon import convert_camel_case_to_lower_case_underscore -from rosidl_pycommon import generate_files - - -def generate_c(generator_arguments_file): - mapping = { - 'idl.h.em': '%s.h', - 'idl__functions.c.em': 'detail/%s__functions.c', - 'idl__functions.h.em': 'detail/%s__functions.h', - 'idl__struct.h.em': 'detail/%s__struct.h', - 'idl__type_support.h.em': 'detail/%s__type_support.h', - } - return generate_files( - generator_arguments_file, mapping, - post_process_callback=prefix_with_bom_if_necessary) - - -def prefix_with_bom_if_necessary(content): - try: - content.encode('ASCII') - except UnicodeError: - prefix = '\ufeff' + \ - '// NOLINT: This file starts with a BOM ' + \ - 'since it contain non-ASCII characters\n' - content = prefix + content - return content - - -BASIC_IDL_TYPES_TO_C = { - 'float': 'float', - 'double': 'double', - 'long double': 'long double', - 'char': 'signed char', - 'wchar': 'uint16_t', - 'boolean': 'bool', - 'octet': 'uint8_t', - 'uint8': 'uint8_t', - 'int8': 'int8_t', - 'uint16': 'uint16_t', - 'int16': 'int16_t', - 'uint32': 'uint32_t', - 'int32': 'int32_t', - 'uint64': 'uint64_t', - 'int64': 'int64_t', +import hashlib +import json +from pathlib import Path +import re +import sys + +from rosidl_parser import definition +from rosidl_parser.parser import parse_idl_file + + +def convert_camel_case_to_lower_case_underscore(value): + # insert an underscore before any upper case letter + # which is followed by a lower case letter + value = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', value) + # insert an underscore before any upper case letter + # which is preseded by a lower case letter or number + value = re.sub('([a-z0-9])([A-Z])', r'\1_\2', value) + return value.lower() + + +def generate_type_hash(generator_arguments_file): + with open(generator_arguments_file, mode='r', encoding='utf-8') as h: + args = json.load(h) + print(args) + + idl_files = {} + idl_files_to_generate = [] + package_name = args['package_name'] + output_dir = args['output_dir'] + + for idl_tuple in args.get('idl_tuples', []): + idl_parts = idl_tuple.rsplit(':', 1) + assert len(idl_parts) == 2 + namespaced_idl_path = str(Path(package_name) / idl_parts[1]) + locator = definition.IdlLocator(*idl_parts) + try: + idl_files[namespaced_idl_path] = parse_idl_file(locator) + idl_files_to_generate.append(namespaced_idl_path) + except Exception as e: + print('Error processing idl file: ' + + str(locator.get_absolute_path()), file=sys.stderr) + raise(e) + + generated_files = [] + for file_key in idl_files_to_generate: + idl_rel_path = Path(file_key) + idl_rel_path = idl_rel_path.relative_to(idl_rel_path.parts[0]) + idl_stem = idl_rel_path.stem + idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) + json_individual_repr = json.dumps(serialize_individual_idl(idl_files[file_key])) + # json_repr = idl_to_hashable_json(file_key, idl_files) + + generate_to_dir = Path(output_dir) / idl_rel_path.parent + generate_to_dir.mkdir(parents=True, exist_ok=True) + + json_path = generate_to_dir / f'{idl_stem}.individual.json' + with json_path.open('w', encoding='utf-8') as json_file: + json_file.write(json_individual_repr) + generated_files.append(str(json_path)) + + json_nested_repr = json.dumps(serialize_nested_idl(file_key, idl_files)) + json_path = generate_to_dir / f'{idl_stem}.json' + with json_path.open('w', encoding='utf-8') as json_file: + json_file.write(json_nested_repr) + generated_files.append(str(json_path)) + + sha = hashlib.sha256() + sha.update(json_nested_repr.encode('utf-8')) + type_hash = sha.hexdigest() + hash_path = generate_to_dir / f'{idl_stem}.sha256' + with hash_path.open('w', encoding='utf-8') as hash_file: + hash_file.write(type_hash) + generated_files.append(str(hash_path)) + + return generated_files + + +# This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg +# TODO There is no FieldType.msg definition for the following rosidl_parser.definition types +# * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long +# * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long +# * +FIELD_TYPES = { + 'nested_type': 0, + 'int8': 1, + 'uint8': 2, + 'int16': 3, + 'uint16': 4, + 'int32': 5, + 'uint32': 6, + 'int64': 7, + 'uint64': 8, + 'float': 9, + 'double': 10, + 'long double': 11, + 'char': 12, + 'wchar': 13, + 'boolean': 14, + 'octet': 15, # byte + definition.UnboundedString: 16, + definition.UnboundedWString: 17, + # TODO there is no rosidl_parser.definition type for fixed strings (there is array of char, though?) + # FIXED_STRING = 18 + # FIXED_WSTRING = 19 + definition.BoundedString: 20, + definition.BoundedWString: 21, } +NESTED_FIELD_TYPE_OFFSETS = { + definition.Array: 32, + definition.BoundedSequence: 64, + definition.UnboundedSequence: 96, +} -def idl_structure_type_to_c_include_prefix(namespaced_type, subdirectory=None): - parts = [ - convert_camel_case_to_lower_case_underscore(x) - for x in (namespaced_type.namespaced_name())] - if subdirectory is not None: - parts[-1:-1] = [subdirectory] - include_prefix = '/'.join(parts) - # Strip service or action suffixes - if include_prefix.endswith('__request'): - include_prefix = include_prefix[:-9] - elif include_prefix.endswith('__response'): - include_prefix = include_prefix[:-10] - if include_prefix.endswith('__goal'): - include_prefix = include_prefix[:-6] - elif include_prefix.endswith('__result'): - include_prefix = include_prefix[:-8] - elif include_prefix.endswith('__feedback'): - include_prefix = include_prefix[:-10] - elif include_prefix.endswith('__send_goal'): - include_prefix = include_prefix[:-11] - elif include_prefix.endswith('__get_result'): - include_prefix = include_prefix[:-12] - return include_prefix - - -def idl_structure_type_to_c_typename(namespaced_type): - return '__'.join(namespaced_type.namespaced_name()) - - -def idl_structure_type_sequence_to_c_typename(namespaced_type): - return idl_structure_type_to_c_typename(namespaced_type) + '__Sequence' - - -def interface_path_to_string(interface_path): - return '/'.join( - list(interface_path.parents[0].parts) + [interface_path.stem]) - - -def idl_declaration_to_c(type_, name): - """ - Convert an IDL type into the C declaration. - - Example input: uint32, std_msgs/String - Example output: uint32_t, char * - - @param type_: The message type - @type type_: rosidl_parser.Type - @param type_: The member name - @type type_: str - """ - if isinstance(type_, AbstractGenericString): - return basetype_to_c(type_) + ' ' + name - if isinstance(type_, Array): - return basetype_to_c(type_.value_type) + ' ' + name + '[' + str(type_.size) + ']' - return idl_type_to_c(type_) + ' ' + name - - -def idl_type_to_c(type_): - if isinstance(type_, Array): - assert False, 'The array size is part of the variable' - if isinstance(type_, AbstractSequence): - if isinstance(type_.value_type, BasicType): - c_type = 'rosidl_runtime_c__' + type_.value_type.typename.replace(' ', '_') - else: - c_type = basetype_to_c(type_.value_type) - c_type += '__Sequence' - return c_type - return basetype_to_c(type_) - - -def basetype_to_c(basetype): - if isinstance(basetype, BasicType): - return BASIC_IDL_TYPES_TO_C[basetype.typename] - if isinstance(basetype, AbstractString): - return 'rosidl_runtime_c__String' - if isinstance(basetype, AbstractWString): - return 'rosidl_runtime_c__U16String' - if isinstance(basetype, NamespacedType): - return idl_structure_type_to_c_typename(basetype) - assert False, str(basetype) - - -def value_to_c(type_, value): - assert isinstance(type_, AbstractType) - assert value is not None - - if isinstance(type_, AbstractString): - return '"%s"' % escape_string(value) - - if isinstance(type_, AbstractWString): - return 'u"%s"' % escape_wstring(value) - - return basic_value_to_c(type_, value) - - -def basic_value_to_c(type_, value): - assert isinstance(type_, BasicType) - assert value is not None - - if 'boolean' == type_.typename: - return 'true' if value else 'false' - - if type_.typename in ( - *CHARACTER_TYPES, - OCTET_TYPE, - 'int8', - 'uint8', - 'int16', - 'uint16', - ): - return str(value) - - if type_.typename == 'int32': - # Handle edge case for INT32_MIN - # Specifically, MSVC is not happy in this case - if -2147483648 == value: - return '({0}l - 1)'.format(value + 1) - return f'{value}l' - - if type_.typename == 'uint32': - return f'{value}ul' - - if type_.typename == 'int64': - # Handle edge case for INT64_MIN - # See https://en.cppreference.com/w/cpp/language/integer_literal - if -9223372036854775808 == value: - return '({0}ll - 1)'.format(value + 1) - return f'{value}ll' - - if type_.typename == 'uint64': - return f'{value}ull' - if 'float' == type_.typename: - return f'{value}f' +def translate_type_id(value_type, offset, result): + if isinstance(value_type, definition.BasicType): + result['type_id'] = FIELD_TYPES[value_type.typename] + offset + elif isinstance(value_type, definition.AbstractGenericString): + result['type_id'] = FIELD_TYPES[type(value_type)] + offset + if value_type.has_maximum_size(): + result['string_length'] = value_type.maximum_size + elif isinstance(value_type, definition.NamespacedType): + result['type_id'] = offset + result['nested_type_name'] = '/'.join(value_type.namespaced_name()) + elif isinstance(value_type, definition.NamedType): + result['type_id'] = offset + result['nested_type_name'] = value_type.name + else: + raise TypeError('Unknown value type ', value_type) + + +def serialize_field_type(ftype: definition.AbstractType): + result = { + 'type_id': 0, + 'length': 0, + 'string_length': 0, + 'nested_type_name': '', + } - if 'double' == type_.typename: - return f'{value}l' + if isinstance(ftype, definition.AbstractNestableType): + translate_type_id(ftype, 0, result) + elif isinstance(ftype, definition.AbstractNestedType): + type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] + translate_type_id(ftype.value_type, type_id_offset, result) + if ftype.has_maximum_size(): + try: + result['length'] = ftype.maximum_size + except AttributeError: + result['length'] = ftype.size + else: + raise Exception('Unable to translate field type', ftype) + + return result + + +def serialize_field(member: definition.Member): + return { + 'name': member.name, + 'type': serialize_field_type(member.type), + # skipping default_value + } - assert False, "unknown basic type '%s'" % type_ +def serialize_individual_type_description(msg: definition.Message): + return { + 'type_name': '/'.join(msg.structure.namespaced_type.namespaced_name()), + 'fields': [serialize_field(member) for member in msg.structure.members] + } -def escape_string(s): - s = s.replace('\\', '\\\\') - s = s.replace('"', r'\"') - return s +def serialize_individual_idl(idl: definition.IdlFile): + for el in idl.content.elements: + if isinstance(el, definition.Message): + return serialize_individual_type_description(el) + elif isinstance(el, definition.Service): + return { + 'request': serialize_individual_type_description(el.request_message), + 'response': serialize_individual_type_description(el.response_message), + } + elif isinstance(el, definition.Action): + # TODO(emersonknapp) + pass + raise Exception("Didn't find something to serialize...") + + +def serialize_nested_idl(file_key, idl_files): + idl = idl_files[file_key] + + includes = [] + referenced_type_descriptions = {} + serialization_data = { + 'type_description': None, + 'referenced_type_descriptions': [], + } -def escape_wstring(s): - return escape_string(s) + for el in idl.content.elements: + if isinstance(el, definition.Include): + includes.append(el.locator) + elif isinstance(el, definition.Message): + serialization_data['type_description'] = serialize_individual_type_description(el) + elif isinstance(el, definition.Service): + serialization_data['type_description'] = { + 'request_message': serialize_individual_type_description(el.request_message), + 'response_message': serialize_individual_type_description(el.response_message), + } + pass + elif isinstance(el, definition.Action): + # TODO + pass + else: + raise Exception(f'Do not know how to hash {el}') + + while includes: + locator = includes.pop() + if locator not in referenced_type_descriptions: + included_file = idl_files[locator] + for el in included_file.content.elements: + if isinstance(el, definition.Include): + includes.append(el.locator) + elif isinstance(el, definition.Message): + referenced_type_descriptions[locator] = serialize_individual_type_description(el) + + referenced_type_descriptions + serialization_data['referenced_type_descriptions'] = sorted( + referenced_type_descriptions.values(), key=lambda td: td['type_name']) + return serialization_data + + +def generate_type_version_hash(file_key, idl_files): + serialized_type_description = idl_to_hashable_json(file_key, idl_files) + # print(json.dumps(serialization_data, indent=2)) + m = hashlib.sha256() + m.update(serialized_type_description.encode('utf-8')) + return m.digest() diff --git a/rosidl_parser/rosidl_parser/type_hash.py b/rosidl_parser/rosidl_parser/type_hash.py deleted file mode 100644 index fb5a5c308..000000000 --- a/rosidl_parser/rosidl_parser/type_hash.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2023 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import json - -from rosidl_parser import definition - -# This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg -# TODO There is no FieldType.msg definition for the following rosidl_parser.definition types -# * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long -# * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long -# * -FIELD_TYPES = { - 'nested_type': 0, - 'int8': 1, - 'uint8': 2, - 'int16': 3, - 'uint16': 4, - 'int32': 5, - 'uint32': 6, - 'int64': 7, - 'uint64': 8, - 'float': 9, - 'double': 10, - 'long double': 11, - 'char': 12, - 'wchar': 13, - 'boolean': 14, - 'octet': 15, # byte - definition.UnboundedString: 16, - definition.UnboundedWString: 17, - # TODO there is no rosidl_parser.definition type for fixed strings (there is array of char, though?) - # FIXED_STRING = 18 - # FIXED_WSTRING = 19 - definition.BoundedString: 20, - definition.BoundedWString: 21, -} - -NESTED_FIELD_TYPE_OFFSETS = { - definition.Array: 32, - definition.BoundedSequence: 64, - definition.UnboundedSequence: 96, -} - -def translate_type_id(value_type, offset, result): - if isinstance(value_type, definition.BasicType): - result['type_id'] = FIELD_TYPES[value_type.typename] + offset - elif isinstance(value_type, definition.AbstractGenericString): - result['type_id'] = FIELD_TYPES[type(value_type)] + offset - if value_type.has_maximum_size(): - result['string_length'] = value_type.maximum_size - elif isinstance(value_type, definition.NamespacedType): - result['type_id'] = offset - result['nested_type_name'] = '/'.join(value_type.namespaced_name()) - elif isinstance(value_type, definition.NamedType): - result['type_id'] = offset - result['nested_type_name'] = value_type.name - else: - raise TypeError('Unknown value type ', value_type) - - -def serialize_field_type(ftype: definition.AbstractType): - result = { - 'type_id': 0, - 'length': 0, - 'string_length': 0, - 'nested_type_name': '', - } - - if isinstance(ftype, definition.AbstractNestableType): - translate_type_id(ftype, 0, result) - elif isinstance(ftype, definition.AbstractNestedType): - type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] - translate_type_id(ftype.value_type, type_id_offset, result) - if ftype.has_maximum_size(): - try: - result['length'] = ftype.maximum_size - except AttributeError: - result['length'] = ftype.size - else: - raise Exception('Unable to translate field type', ftype) - - return result - - -def serialize_field(member: definition.Member): - return { - 'name': member.name, - 'type': serialize_field_type(member.type), - # skipping default_value - } - - -def serialize_individual_type_description(msg: definition.Message): - return { - 'type_name': '/'.join(msg.structure.namespaced_type.namespaced_name()), - 'fields': [serialize_field(member) for member in msg.structure.members] - } - - -def idl_to_hashable_json(file_key, idl_files): - idl = idl_files[file_key] - - includes = [] - referenced_type_descriptions = {} - serialization_data = { - 'type_description': None, - 'referenced_type_descriptions': [], - } - - for el in idl.content.elements: - if isinstance(el, definition.Include): - includes.append(el.locator) - elif isinstance(el, definition.Message): - serialization_data['type_description'] = serialize_individual_type_description(el) - elif isinstance(el, definition.Service): - serialization_data['type_description'] = { - 'request_message': serialize_individual_type_description(el.request_message), - 'response_message': serialize_individual_type_description(el.response_message), - } - pass - elif isinstance(el, definition.Action): - # TODO - pass - else: - raise Exception(f'Do not know how to hash {el}') - - while includes: - locator = includes.pop() - if locator not in referenced_type_descriptions: - included_file = idl_files[locator] - for el in included_file.content.elements: - if isinstance(el, definition.Include): - includes.append(el.locator) - elif isinstance(el, definition.Message): - referenced_type_descriptions[locator] = serialize_individual_type_description(el) - - referenced_type_descriptions - serialization_data['referenced_type_descriptions'] = sorted( - referenced_type_descriptions.values(), key=lambda td: td['type_name']) - return json.dumps(serialization_data) - - -def generate_type_version_hash(file_key, idl_files): - serialized_type_description = idl_to_hashable_json(file_key, idl_files) - # print(json.dumps(serialization_data, indent=2)) - m = hashlib.sha256() - m.update(serialized_type_description.encode('utf-8')) - return m.digest() diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 0554a6c5a..1963403c2 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -23,9 +23,6 @@ import em from rosidl_parser.definition import IdlLocator from rosidl_parser.parser import parse_idl_file -from rosidl_parser.type_hash import generate_type_version_hash -from rosidl_parser.type_hash import idl_to_hashable_json - def convert_camel_case_to_lower_case_underscore(value): # insert an underscore before any upper case letter @@ -110,17 +107,8 @@ def generate_files( if not keep_case: idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) - # Generate hashable representation and write to file - json_repr = idl_to_hashable_json(file_key, idl_files) - # TODO(emersonknapp) helper fn? - json_file = pathlib.Path(args['output_dir']) / idl_rel_path.parent / f'{idl_stem}.json' - json_file.parent.mkdir(parents=True, exist_ok=True) - with json_file.open('w', encoding='utf-8') as f: - f.write(json_repr) - - # Create hash + # TODO(emersonknapp) load hash from generator_type_hash outputs m = hashlib.sha256() - m.update(json_repr.encode('utf-8')) type_hash = m.digest() # Run codegen for files From fcc5bd8ff45b69aa887df1ffab7e45cc6fe082b6 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Fri, 3 Feb 2023 17:12:38 -0800 Subject: [PATCH 10/65] Cleanup some C artifacts Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash-extras.cmake | 6 -- .../rosidl_generator_type_hash/cli.py | 81 ------------------- rosidl_generator_type_hash/setup.cfg | 2 +- 3 files changed, 1 insertion(+), 88 deletions(-) delete mode 100644 rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake delete mode 100644 rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake deleted file mode 100644 index 8a18b8053..000000000 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake +++ /dev/null @@ -1,6 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/register_type_hash.cmake") -rosidl_generator_type_hash_extras( - "${rosidl_generator_type_hash_DIR}/../../../lib/rosidl_generator_type_hash/rosidl_generator_type_hash" - "${rosidl_generator_type_hash_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_hash/__init__.py" - "${rosidl_generator_type_hash_DIR}/../resource" -) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py deleted file mode 100644 index f6c786d7d..000000000 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/cli.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2021 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pathlib - -from ament_index_python import get_package_share_directory -from rosidl_cli.command.generate.extensions import GenerateCommandExtension -from rosidl_cli.command.helpers import generate_visibility_control_file -from rosidl_cli.command.helpers import legacy_generator_arguments_file -from rosidl_cli.command.translate.api import translate - -from rosidl_generator_c import generate_c - - -class GenerateC(GenerateCommandExtension): - - def generate( - self, - package_name, - interface_files, - include_paths, - output_path - ): - generated_files = [] - - package_share_path = \ - pathlib.Path(get_package_share_directory('rosidl_generator_c')) - templates_path = package_share_path / 'resource' - - # Normalize interface definition format to .idl - idl_interface_files = [] - non_idl_interface_files = [] - for path in interface_files: - if not path.endswith('.idl'): - non_idl_interface_files.append(path) - else: - idl_interface_files.append(path) - if non_idl_interface_files: - idl_interface_files.extend(translate( - package_name=package_name, - interface_files=non_idl_interface_files, - include_paths=include_paths, - output_format='idl', - output_path=output_path / 'tmp', - )) - - # Generate visibility control file - visibility_control_file_template_path = \ - templates_path / 'rosidl_generator_c__visibility_control.h.in' - visibility_control_file_path = \ - output_path / 'msg' / 'rosidl_generator_c__visibility_control.h' - - generate_visibility_control_file( - package_name=package_name, - template_path=visibility_control_file_template_path, - output_path=visibility_control_file_path - ) - generated_files.append(visibility_control_file_path) - - # Generate code - with legacy_generator_arguments_file( - package_name=package_name, - interface_files=idl_interface_files, - include_paths=include_paths, - templates_path=templates_path, - output_path=output_path - ) as path_to_arguments_file: - generated_files.extend(generate_c(path_to_arguments_file)) - - return generated_files diff --git a/rosidl_generator_type_hash/setup.cfg b/rosidl_generator_type_hash/setup.cfg index a269e3416..b62f1af69 100644 --- a/rosidl_generator_type_hash/setup.cfg +++ b/rosidl_generator_type_hash/setup.cfg @@ -1,3 +1,3 @@ [options.entry_points] rosidl_cli.command.generate.type_extensions = - c = rosidl_generator_type_hash.cli:GenerateTypeHash + type_hash = rosidl_generator_type_hash.cli:GenerateTypeHash From 90440942b20db8fb1fea393e1121c6b1a17124be Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 6 Feb 2023 17:59:51 -0800 Subject: [PATCH 11/65] rosidl generator creating and installing expected files Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 2 - .../resource/srv__struct.hpp.em | 2 +- .../bin/rosidl_generator_type_hash | 17 +- .../cmake/register_type_hash.cmake | 16 -- ...erator_type_hash_generate_interfaces.cmake | 108 +++++----- ...rosidl_generator_type_hash-extras.cmake.in | 27 ++- .../rosidl_generator_type_hash/__init__.py | 184 +++++++++--------- 7 files changed, 169 insertions(+), 187 deletions(-) delete mode 100644 rosidl_generator_type_hash/cmake/register_type_hash.cmake diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index 5fb9ebc85..304a71aaa 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -32,7 +32,6 @@ extern "C" #include #include - @####################################################################### @# Handle message @####################################################################### @@ -153,7 +152,6 @@ TEMPLATE( TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.feedback_message, include_directives=include_directives) message=action.feedback_message, include_directives=include_directives, type_hash=type_hash) }@ diff --git a/rosidl_generator_cpp/resource/srv__struct.hpp.em b/rosidl_generator_cpp/resource/srv__struct.hpp.em index c22618b30..c18a11547 100644 --- a/rosidl_generator_cpp/resource/srv__struct.hpp.em +++ b/rosidl_generator_cpp/resource/srv__struct.hpp.em @@ -17,7 +17,7 @@ TEMPLATE( TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=service.event_message, include_directives=include_directives) + message=service.event_message, include_directives=include_directives, type_hash=type_hash) }@ @[for ns in service.namespaced_type.namespaces]@ diff --git a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash index 277f99a7f..63f4bb8e9 100755 --- a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash +++ b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash @@ -26,12 +26,21 @@ def main(argv=sys.argv[1:]): description='Generate hashable representations and hashes of ROS interfaces.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--generator-arguments-file', - required=True, - help='The location of the file containing the generator arguments') + '--package-name', required=True, help='TODO(emersonknapp') + parser.add_argument( + '--output-dir', required=True, help='TODO(emersonknapp') + parser.add_argument( + '--idl-tuples', nargs='+', required=True, help='TODO(emersonknapp)') + parser.add_argument( + '--include-paths', nargs='*', required=False, help='TODO(emersonknapp)') + args = parser.parse_args(argv) - generate_type_hash(args.generator_arguments_file) + generate_type_hash( + package_name=args.package_name, + output_dir=args.output_dir, + idl_tuples=args.idl_tuples, + include_paths=args.include_paths) if __name__ == '__main__': diff --git a/rosidl_generator_type_hash/cmake/register_type_hash.cmake b/rosidl_generator_type_hash/cmake/register_type_hash.cmake deleted file mode 100644 index d448d1df5..000000000 --- a/rosidl_generator_type_hash/cmake/register_type_hash.cmake +++ /dev/null @@ -1,16 +0,0 @@ -macro(rosidl_generator_type_hash_extras BIN GENERATOR_FILES TEMPLATE_DIR) - find_package(ament_cmake_core QUIET REQUIRED) - ament_register_extension( - "rosidl_generate_idl_interfaces" - "rosidl_generator_type_hash" - "rosidl_generator_type_hash_generate_interfaces.cmake") - - normalize_path(BIN "${BIN}") - set(rosidl_generator_type_hash_BIN "${BIN}") - - normalize_path(GENERATOR_FILES "${GENERATOR_FILES}") - set(rosidl_generator_type_hash_GENERATOR_FILES "${GENERATOR_FILES}") - - normalize_path(TEMPLATE_DIR "${TEMPLATE_DIR}") - set(rosidl_generator_type_hash_TEMPLATE_DIR "${TEMPLATE_DIR}") -endmacro() diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index 42027e822..c507e56b3 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -1,4 +1,4 @@ -# Copyright 2015-2018 Open Source Robotics Foundation, Inc. +# Copyright 2023 Open Source Robotics Foundation, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,35 +12,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(rosidl_generate_interfaces_type_hash_IDL_TUPLES - ${rosidl_generate_interfaces_IDL_TUPLES}) +find_package(Python3 REQUIRED COMPONENTS Interpreter) + set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash/${PROJECT_NAME}") -set(_generated_jsons "") -set(_generated_hashes "") +set(_generated_json_in "") +set(_generated_files "") foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) get_filename_component(_parent_folder "${_abs_idl_file}" DIRECTORY) get_filename_component(_parent_folder "${_parent_folder}" NAME) - get_filename_component(_idl_name "${_abs_idl_file}" NAME_WE) - string_camel_case_to_lower_case_underscore("${_idl_name}" _idl_stem) + get_filename_component(_idl_stem "${_abs_idl_file}" NAME_WE) + # string_camel_case_to_lower_case_underscore("${_idl_name}" _idl_stem) + list(APPEND _generated_json_in + "${_output_path}/${_parent_folder}/${_idl_stem}.json.in") + list(APPEND _generated_files + "${_output_path}/${_parent_folder}/${_idl_stem}.json") list(APPEND _generated_files - "${_output_path}/${_parent_folder}/${_idl_stem}.json" - ) - list(APPEND _generated_hashes - "${_output_path}/${_parent_folder}/${_idl_stem}.sha256" - ) + "${_output_path}/${_parent_folder}/${_idl_stem}.sha256") endforeach() set(_dependency_files "") +set(_dependency_paths "") set(_dependencies "") foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) - foreach(_idl_file ${${_pkg_name}_IDL_FILES}) - set(_abs_idl_file "${${_pkg_name}_DIR}/../${_idl_file}") - normalize_path(_abs_idl_file "${_abs_idl_file}") - list(APPEND _dependency_files "${_abs_idl_file}") - list(APPEND _dependencies "${_pkg_name}:${_abs_idl_file}") + set(_include_path "${${_pkg_name}_DIR}/..") + normalize_path(_include_path "${_include_path}") + list(APPEND _dependency_paths "${_pkg_name}:${_include_path}") + foreach(_json_in_file ${${_pkg_name}_JSON_IN_FILES}) + set(_abs_json_in_file ${_include_path}/${_json_in_file}) + list(APPEND _dependency_files ${_abs_json_in_file}) endforeach() endforeach() +set(${rosidl_generate_interfaces_TARGET}__JSON_IN_FILES ${_generated_json_in}) + set(target_dependencies "${rosidl_generator_type_hash_BIN}" ${rosidl_generator_type_hash_GENERATOR_FILES} @@ -52,68 +56,44 @@ foreach(dep ${target_dependencies}) endif() endforeach() -set(generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash__arguments.json") -rosidl_write_generator_arguments( - "${generator_arguments_file}" - PACKAGE_NAME "${PROJECT_NAME}" - IDL_TUPLES "${rosidl_generate_interfaces_type_hash_IDL_TUPLES}" - ROS_INTERFACE_DEPENDENCIES "${_dependencies}" - OUTPUT_DIR "${_output_path}" - TEMPLATE_DIR "${rosidl_generator_type_hash_TEMPLATE_DIR}" - TARGET_DEPENDENCIES ${target_dependencies} -) - -find_package(Python3 REQUIRED COMPONENTS Interpreter) - +set(_generated_files "${_generated_files};${_generated_json_in}") add_custom_command( - OUTPUT ${_generated_jsons} ${_generated_hashes} COMMAND Python3::Interpreter - ARGS ${rosidl_generator_type_hash_BIN} - --generator-arguments-file "${generator_arguments_file}" + ARGS + ${rosidl_generator_type_hash_BIN} + --package-name "${PROJECT_NAME}" + --output-dir "${_output_path}" + --idl-tuples ${rosidl_generate_interfaces_IDL_TUPLES} + --include-paths ${_dependency_paths} + OUTPUT ${_generated_files} DEPENDS ${target_dependencies} COMMENT "Generating type hashes for ROS interfaces" VERBATIM ) -set(_target_suffix "__rosidl_generator_type_hash") - -add_custom_target( - ${rosidl_generate_interfaces_TARGET}${_target_suffix} - DEPENDS ${_generated_jsons} ${_generated_hashes}) -# add_custom_target(${PROJECT_NAME}::${rosidl_generate_interfaces_TARGET}${_target_suffix} ALIAS -# ${rosidl_generate_interfaces_TARGET}${_target_suffix}) -if(rosidl_generate_interfaces_LIBRARY_NAME) - set_target_properties(${rosidl_generate_interfaces_TARGET}${_target_suffix} - PROPERTIES OUTPUT_NAME "${rosidl_generate_interfaces_LIBRARY_NAME}${_target_suffix}") -endif() -# foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) -# # Depend on targets generated by this generator in dependency packages -# target_link_libraries( -# ${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBLIC -# ${${_pkg_name}_TARGETS${_target_suffix}}) -# endforeach() +set(_target "${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_hash") +add_custom_target(${_target} DEPENDS ${_generated_files}) # # Make top level generation target depend on this generated library -add_dependencies( - ${rosidl_generate_interfaces_TARGET} - ${rosidl_generate_interfaces_TARGET}${_target_suffix} -) +add_dependencies(${rosidl_generate_interfaces_TARGET} ${_target}) if(NOT rosidl_generate_interfaces_SKIP_INSTALL) - install( - DIRECTORY ${_output_path}/ - DESTINATION "include/${PROJECT_NAME}/${PROJECT_NAME}" - PATTERN "*.json" - ) + foreach(_generated_file ${_generated_files}) + get_filename_component(_parent_folder "${_generated_file}" DIRECTORY) + get_filename_component(_parent_folder "${_parent_folder}" NAME) + install( + FILES ${_generated_file} + DESTINATION "share/${PROJECT_NAME}/${_parent_folder}") + endforeach() # Export old-style CMake variables - ament_export_include_directories("include/${PROJECT_NAME}") - ament_export_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix}) + # ament_export_include_directories("include/${PROJECT_NAME}") + # ament_export_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix}) # Export modern CMake targets - ament_export_targets(export_${rosidl_generate_interfaces_TARGET}${_target_suffix}) - rosidl_export_typesupport_targets(${_target_suffix} - ${rosidl_generate_interfaces_TARGET}${_target_suffix}) + # ament_export_targets(export_${rosidl_generate_interfaces_TARGET}${_target_suffix}) + # rosidl_export_typesupport_targets(${_target_suffix} + # ${rosidl_generate_interfaces_TARGET}${_target_suffix}) # install( # TARGETS ${rosidl_generate_interfaces_TARGET}${_target_suffix} diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in index 8a18b8053..95b4084a0 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in @@ -1,6 +1,21 @@ -include("${CMAKE_CURRENT_LIST_DIR}/register_type_hash.cmake") -rosidl_generator_type_hash_extras( - "${rosidl_generator_type_hash_DIR}/../../../lib/rosidl_generator_type_hash/rosidl_generator_type_hash" - "${rosidl_generator_type_hash_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_hash/__init__.py" - "${rosidl_generator_type_hash_DIR}/../resource" -) +find_package(ament_cmake_core QUIET REQUIRED) + +ament_register_extension( + "rosidl_generate_idl_interfaces" + "rosidl_generator_type_hash" + "rosidl_generator_type_hash_generate_interfaces.cmake") + +set(rosidl_generator_type_hash_BIN + "${rosidl_generator_type_hash_DIR}/../../../lib/rosidl_generator_type_hash/rosidl_generator_type_hash") +normalize_path(rosidl_generator_type_hash_BIN + "${rosidl_generator_type_hash_BIN}") + +set(rosidl_generator_type_hash_GENERATOR_FILES + "${rosidl_generator_type_hash_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_hash/__init__.py") +normalize_path(rosidl_generator_type_hash_GENERATOR_FILES + "${rosidl_generator_type_hash_GENERATOR_FILES}") + +set(rosidl_generator_type_hash_TEMPLATE_DIR + "${rosidl_generator_type_hash_DIR}/../resource") +normalize_path(rosidl_generator_type_hash_TEMPLATE_DIR + "${rosidl_generator_type_hash_TEMPLATE_DIR}") diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index fabab24e3..4e540829c 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -15,73 +15,70 @@ import hashlib import json from pathlib import Path -import re import sys +from typing import List from rosidl_parser import definition from rosidl_parser.parser import parse_idl_file -def convert_camel_case_to_lower_case_underscore(value): - # insert an underscore before any upper case letter - # which is followed by a lower case letter - value = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', value) - # insert an underscore before any upper case letter - # which is preseded by a lower case letter or number - value = re.sub('([a-z0-9])([A-Z])', r'\1_\2', value) - return value.lower() - - -def generate_type_hash(generator_arguments_file): - with open(generator_arguments_file, mode='r', encoding='utf-8') as h: - args = json.load(h) - print(args) - - idl_files = {} - idl_files_to_generate = [] - package_name = args['package_name'] - output_dir = args['output_dir'] +def generate_type_hash( + package_name: str, + output_dir: str, + idl_tuples: List[str], + include_paths: List[str] +): + include_map = { + package_name: Path(output_dir) + } + for include_tuple in include_paths: + include_parts = include_tuple.rsplit(':', 1) + assert len(include_parts) == 2 + include_package_name, include_base_path = include_parts + include_map[include_package_name] = Path(include_base_path) - for idl_tuple in args.get('idl_tuples', []): + generated_files = [] + json_contents = [] + for idl_tuple in idl_tuples: + print(f'Generating for {idl_tuple}') idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 - namespaced_idl_path = str(Path(package_name) / idl_parts[1]) locator = definition.IdlLocator(*idl_parts) try: - idl_files[namespaced_idl_path] = parse_idl_file(locator) - idl_files_to_generate.append(namespaced_idl_path) + idl_file = parse_idl_file(locator) except Exception as e: - print('Error processing idl file: ' + + print('Error processing idl file: ' + str(locator.get_absolute_path()), file=sys.stderr) raise(e) - generated_files = [] - for file_key in idl_files_to_generate: - idl_rel_path = Path(file_key) - idl_rel_path = idl_rel_path.relative_to(idl_rel_path.parts[0]) + idl_rel_path = Path(idl_parts[1]) idl_stem = idl_rel_path.stem - idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) - json_individual_repr = json.dumps(serialize_individual_idl(idl_files[file_key])) - # json_repr = idl_to_hashable_json(file_key, idl_files) - - generate_to_dir = Path(output_dir) / idl_rel_path.parent + generate_to_dir = (Path(output_dir) / idl_rel_path).parent generate_to_dir.mkdir(parents=True, exist_ok=True) + stem_path = generate_to_dir / f'{idl_stem}' + + json_in = generate_json_in(idl_file) + # Need to generate all .json.in before expanding .json final + json_contents.append((stem_path, json_in)) - json_path = generate_to_dir / f'{idl_stem}.individual.json' + json_path = stem_path.with_suffix('.json.in') with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json_individual_repr) - generated_files.append(str(json_path)) + json_file.write(json.dumps(json_in, indent=2)) + generated_files.append(str(json_path)) - json_nested_repr = json.dumps(serialize_nested_idl(file_key, idl_files)) - json_path = generate_to_dir / f'{idl_stem}.json' + for stem_path, json_in in json_contents: + json_out = generate_json_out(json_in, include_map) + json_out_repr = json.dumps(json_out) + + json_path = stem_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json_nested_repr) - generated_files.append(str(json_path)) + json_file.write(json_out_repr) + generated_files.append(str(json_path)) sha = hashlib.sha256() - sha.update(json_nested_repr.encode('utf-8')) + sha.update(json_out_repr.encode('utf-8')) type_hash = sha.hexdigest() - hash_path = generate_to_dir / f'{idl_stem}.sha256' + hash_path = stem_path.with_suffix('.sha256') with hash_path.open('w', encoding='utf-8') as hash_file: hash_file.write(type_hash) generated_files.append(str(hash_path)) @@ -183,67 +180,66 @@ def serialize_individual_type_description(msg: definition.Message): } -def serialize_individual_idl(idl: definition.IdlFile): - for el in idl.content.elements: - if isinstance(el, definition.Message): - return serialize_individual_type_description(el) - elif isinstance(el, definition.Service): - return { - 'request': serialize_individual_type_description(el.request_message), - 'response': serialize_individual_type_description(el.response_message), - } - elif isinstance(el, definition.Action): - # TODO(emersonknapp) - pass - raise Exception("Didn't find something to serialize...") - - -def serialize_nested_idl(file_key, idl_files): - idl = idl_files[file_key] - +def generate_json_in(idl: definition.IdlFile): + type_description = None includes = [] - referenced_type_descriptions = {} - serialization_data = { - 'type_description': None, - 'referenced_type_descriptions': [], - } - for el in idl.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) elif isinstance(el, definition.Message): - serialization_data['type_description'] = serialize_individual_type_description(el) + type_description = serialize_individual_type_description(el) elif isinstance(el, definition.Service): - serialization_data['type_description'] = { + type_description = { 'request_message': serialize_individual_type_description(el.request_message), 'response_message': serialize_individual_type_description(el.response_message), } - pass elif isinstance(el, definition.Action): - # TODO - pass + # TODO(emersonknapp) + type_description = { + 'goal_service': { + 'request_message': None, + 'response_message': None, + }, + 'result_service': { + 'request_message': None, + 'response_message': None, + }, + 'feedback_message': None, + } else: raise Exception(f'Do not know how to hash {el}') + if type_description is None: + raise Exception('Did not find an interface to serialize in IDL file') - while includes: - locator = includes.pop() - if locator not in referenced_type_descriptions: - included_file = idl_files[locator] - for el in included_file.content.elements: - if isinstance(el, definition.Include): - includes.append(el.locator) - elif isinstance(el, definition.Message): - referenced_type_descriptions[locator] = serialize_individual_type_description(el) - - referenced_type_descriptions - serialization_data['referenced_type_descriptions'] = sorted( - referenced_type_descriptions.values(), key=lambda td: td['type_name']) - return serialization_data - - -def generate_type_version_hash(file_key, idl_files): - serialized_type_description = idl_to_hashable_json(file_key, idl_files) - # print(json.dumps(serialization_data, indent=2)) - m = hashlib.sha256() - m.update(serialized_type_description.encode('utf-8')) - return m.digest() + includes = [ + str(Path(include).with_suffix('.json.in')) for include in includes + ] + return { + 'type_description': type_description, + 'includes': includes, + } + + +def generate_json_out(json_in, includes_map): + pending_includes = json_in['includes'] + loaded_includes = {} + while pending_includes: + process_include = pending_includes.pop() + if process_include in loaded_includes: + continue + p_path = Path(process_include) + assert(not p_path.is_absolute()) + include_package = p_path.parts[0] + include_file = includes_map[include_package] / p_path.relative_to(include_package) + + with include_file.open('r') as include_file: + include_json = json.load(include_file) + + loaded_includes[process_include] = include_json['type_description'] + pending_includes.extend(include_json['includes']) + + return { + 'type_description': json_in['type_description'], + 'referenced_type_descriptions': sorted( + loaded_includes.values(), key=lambda td: td['type_name']) + } From 8b5d892bdd112f721f6f9d3db84edf0d41ac5546 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 6 Feb 2023 18:42:47 -0800 Subject: [PATCH 12/65] Revert pycommon changes Signed-off-by: Emerson Knapp --- rosidl_pycommon/rosidl_pycommon/__init__.py | 84 ++++++--------------- 1 file changed, 24 insertions(+), 60 deletions(-) diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 1963403c2..8cd10dd85 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib from io import StringIO import json import os @@ -24,6 +23,7 @@ from rosidl_parser.definition import IdlLocator from rosidl_parser.parser import parse_idl_file + def convert_camel_case_to_lower_case_underscore(value): # insert an underscore before any upper case letter # which is followed by a lower case letter @@ -62,76 +62,40 @@ def generate_files( latest_target_timestamp = get_newest_modification_time(args['target_dependencies']) generated_files = [] - idl_files_to_generate = [] - idl_files = {} - package_name = args['package_name'] - for idl_tuple in args.get('idl_tuples', []): idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 - namespaced_idl_path = str(pathlib.Path(package_name) / idl_parts[1]) - locator = IdlLocator(*idl_parts) + idl_rel_path = pathlib.Path(idl_parts[1]) + idl_stem = idl_rel_path.stem + if not keep_case: + idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) try: - idl_files[namespaced_idl_path] = parse_idl_file(locator) - idl_files_to_generate.append(namespaced_idl_path) - except Exception as e: - print( - 'Error processing idl file: ' + - str(locator.get_absolute_path()), file=sys.stderr) - raise(e) - - for interface_dep in args.get('ros_interface_dependencies', []): - tuple_parts = interface_dep.rsplit(':', 1) - assert len(tuple_parts) == 2 - referenced_package_name, idl_abs_path = tuple_parts - - base_path, pkg, rel_path = idl_abs_path.rpartition(referenced_package_name) - assert pkg == referenced_package_name - namespaced_idl_path = pkg + rel_path - - # rel_path it starts with a pathsep from rpartition - locator = IdlLocator(base_path + pkg, rel_path[1:]) - try: - idl_files[namespaced_idl_path] = parse_idl_file(locator) + idl_file = parse_idl_file(locator) + for template_file, generated_filename in mapping.items(): + generated_file = os.path.join( + args['output_dir'], str(idl_rel_path.parent), + generated_filename % idl_stem) + generated_files.append(generated_file) + data = { + 'package_name': args['package_name'], + 'interface_path': idl_rel_path, + 'content': idl_file.content, + 'type_hash': b'\0'*32, + } + if additional_context is not None: + data.update(additional_context) + expand_template( + os.path.basename(template_file), data, + generated_file, minimum_timestamp=latest_target_timestamp, + template_basepath=template_basepath, + post_process_callback=post_process_callback) except Exception as e: print( 'Error processing idl file: ' + str(locator.get_absolute_path()), file=sys.stderr) raise(e) - for file_key in idl_files_to_generate: - idl_rel_path = pathlib.Path(file_key) - idl_rel_path = idl_rel_path.relative_to(idl_rel_path.parts[0]) - idl_stem = idl_rel_path.stem - if not keep_case: - idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) - - # TODO(emersonknapp) load hash from generator_type_hash outputs - m = hashlib.sha256() - type_hash = m.digest() - - # Run codegen for files - idl_file = idl_files[file_key] - for template_file, generated_filename in mapping.items(): - generated_file = os.path.join( - args['output_dir'], str(idl_rel_path.parent), - generated_filename % idl_stem) - generated_files.append(generated_file) - data = { - 'package_name': package_name, - 'interface_path': idl_rel_path, - 'content': idl_file.content, - 'type_hash': type_hash, - } - if additional_context is not None: - data.update(additional_context) - expand_template( - os.path.basename(template_file), data, - generated_file, minimum_timestamp=latest_target_timestamp, - template_basepath=template_basepath, - post_process_callback=post_process_callback) - return generated_files From 7e79cee6f9474e592203d8a354e83b8937f0478f Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Feb 2023 12:47:25 -0800 Subject: [PATCH 13/65] Generate action struct Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 4e540829c..f0617e624 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -40,7 +40,6 @@ def generate_type_hash( generated_files = [] json_contents = [] for idl_tuple in idl_tuples: - print(f'Generating for {idl_tuple}') idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 locator = definition.IdlLocator(*idl_parts) @@ -90,7 +89,6 @@ def generate_type_hash( # TODO There is no FieldType.msg definition for the following rosidl_parser.definition types # * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long # * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long -# * FIELD_TYPES = { 'nested_type': 0, 'int8': 1, @@ -110,7 +108,7 @@ def generate_type_hash( 'octet': 15, # byte definition.UnboundedString: 16, definition.UnboundedWString: 17, - # TODO there is no rosidl_parser.definition type for fixed strings (there is array of char, though?) + # TODO there is no rosidl_parser.definition type for fixed strings (caveat: array of char) # FIXED_STRING = 18 # FIXED_WSTRING = 19 definition.BoundedString: 20, @@ -183,6 +181,7 @@ def serialize_individual_type_description(msg: definition.Message): def generate_json_in(idl: definition.IdlFile): type_description = None includes = [] + # TODO(emersonknapp): do the individual message types get their own type hash? for el in idl.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) @@ -194,17 +193,20 @@ def generate_json_in(idl: definition.IdlFile): 'response_message': serialize_individual_type_description(el.response_message), } elif isinstance(el, definition.Action): - # TODO(emersonknapp) type_description = { - 'goal_service': { - 'request_message': None, - 'response_message': None, + 'send_goal_service': { + 'request_message': serialize_individual_type_description( + el.send_goal_service.request_message), + 'response_message': serialize_individual_type_description( + el.send_goal_service.response_message), }, - 'result_service': { - 'request_message': None, - 'response_message': None, + 'get_result_service': { + 'request_message': serialize_individual_type_description( + el.get_result_service.request_message), + 'response_message': serialize_individual_type_description( + el.get_result_service.response_message), }, - 'feedback_message': None, + 'feedback_message': serialize_individual_type_description(el.feedback_message), } else: raise Exception(f'Do not know how to hash {el}') From 7dde0c2350ca94b320239264e1213ce893211573 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Feb 2023 15:49:23 -0800 Subject: [PATCH 14/65] Type hash depended upon and loaded by C and C++ generators Signed-off-by: Emerson Knapp --- .../rosidl_write_generator_arguments.cmake | 1 + rosidl_generator_c/cmake/register_c.cmake | 1 + ...sidl_generator_c_generate_interfaces.cmake | 4 ++++ rosidl_generator_c/package.xml | 1 + rosidl_generator_cpp/cmake/register_cpp.cmake | 1 + ...dl_generator_cpp_generate_interfaces.cmake | 4 ++++ rosidl_generator_cpp/package.xml | 1 + ...erator_type_hash_generate_interfaces.cmake | 16 +++++++------- rosidl_generator_type_hash/package.xml | 2 -- rosidl_pycommon/rosidl_pycommon/__init__.py | 21 ++++++++++++++++++- 10 files changed, 42 insertions(+), 10 deletions(-) diff --git a/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake b/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake index 33a3733ea..5b5e03098 100644 --- a/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake +++ b/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake @@ -32,6 +32,7 @@ function(rosidl_write_generator_arguments output_file) set(OPTIONAL_MULTI_VALUE_KEYWORDS "ROS_INTERFACE_DEPENDENCIES" # since the dependencies can be empty "TARGET_DEPENDENCIES" + "TYPE_HASH_TUPLES" "ADDITIONAL_FILES") cmake_parse_arguments( diff --git a/rosidl_generator_c/cmake/register_c.cmake b/rosidl_generator_c/cmake/register_c.cmake index e61455d12..36b1f307a 100644 --- a/rosidl_generator_c/cmake/register_c.cmake +++ b/rosidl_generator_c/cmake/register_c.cmake @@ -14,6 +14,7 @@ macro(rosidl_generator_c_extras BIN GENERATOR_FILES TEMPLATE_DIR) find_package(ament_cmake_core QUIET REQUIRED) + find_package(rosidl_generator_type_hash QUIET REQUIRED) ament_register_extension( "rosidl_generate_idl_interfaces" "rosidl_generator_c" diff --git a/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake b/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake index 11652ea9b..7d44991a4 100644 --- a/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake +++ b/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake @@ -83,6 +83,7 @@ rosidl_write_generator_arguments( OUTPUT_DIR "${_output_path}" TEMPLATE_DIR "${rosidl_generator_c_TEMPLATE_DIR}" TARGET_DEPENDENCIES ${target_dependencies} + TYPE_HASH_TUPLES "${${rosidl_generate_interfaces_TARGET}__HASH_TUPLES}" ) find_package(Python3 REQUIRED COMPONENTS Interpreter) @@ -142,6 +143,9 @@ target_link_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBL rosidl_runtime_c::rosidl_runtime_c rosidl_typesupport_interface::rosidl_typesupport_interface rcutils::rcutils) +add_dependencies( + ${rosidl_generate_interfaces_TARGET}${_target_suffix} + ${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_hash) # Make top level generation target depend on this generated library add_dependencies( diff --git a/rosidl_generator_c/package.xml b/rosidl_generator_c/package.xml index f76ef4c49..0d1cad9e6 100644 --- a/rosidl_generator_c/package.xml +++ b/rosidl_generator_c/package.xml @@ -32,6 +32,7 @@ rosidl_cli rosidl_parser rcutils + rosidl_generator_type_hash ament_lint_auto ament_lint_common diff --git a/rosidl_generator_cpp/cmake/register_cpp.cmake b/rosidl_generator_cpp/cmake/register_cpp.cmake index df16961d3..afdf2c4ec 100644 --- a/rosidl_generator_cpp/cmake/register_cpp.cmake +++ b/rosidl_generator_cpp/cmake/register_cpp.cmake @@ -14,6 +14,7 @@ macro(rosidl_generator_cpp_extras BIN GENERATOR_FILES TEMPLATE_DIR) find_package(ament_cmake_core QUIET REQUIRED) + find_package(rosidl_generator_type_hash QUIET REQUIRED) ament_register_extension( "rosidl_generate_idl_interfaces" "rosidl_generator_cpp" diff --git a/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake b/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake index 9e8f9f546..3046a8e70 100644 --- a/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake +++ b/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake @@ -75,6 +75,7 @@ rosidl_write_generator_arguments( OUTPUT_DIR "${_output_path}" TEMPLATE_DIR "${rosidl_generator_cpp_TEMPLATE_DIR}" TARGET_DEPENDENCIES ${target_dependencies} + TYPE_HASH_TUPLES "${${rosidl_generate_interfaces_TARGET}__HASH_TUPLES}" ) find_package(Python3 REQUIRED COMPONENTS Interpreter) @@ -99,6 +100,9 @@ add_custom_target( DEPENDS ${_generated_headers} ) +add_dependencies( + ${rosidl_generate_interfaces_TARGET}__cpp + ${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_hash) set(_target_suffix "__rosidl_generator_cpp") add_library(${rosidl_generate_interfaces_TARGET}${_target_suffix} INTERFACE) diff --git a/rosidl_generator_cpp/package.xml b/rosidl_generator_cpp/package.xml index ae6c1cc36..51c31730e 100644 --- a/rosidl_generator_cpp/package.xml +++ b/rosidl_generator_cpp/package.xml @@ -28,6 +28,7 @@ ament_index_python rosidl_cli + rosidl_generator_type_hash rosidl_parser ament_lint_auto diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index c507e56b3..d4cab6970 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -16,18 +16,20 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash/${PROJECT_NAME}") set(_generated_json_in "") -set(_generated_files "") +set(_generated_json "") +set(_generated_hash_tuples "") +set(_generated_hash_files "") foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) get_filename_component(_parent_folder "${_abs_idl_file}" DIRECTORY) get_filename_component(_parent_folder "${_parent_folder}" NAME) get_filename_component(_idl_stem "${_abs_idl_file}" NAME_WE) - # string_camel_case_to_lower_case_underscore("${_idl_name}" _idl_stem) list(APPEND _generated_json_in "${_output_path}/${_parent_folder}/${_idl_stem}.json.in") - list(APPEND _generated_files + list(APPEND _generated_json "${_output_path}/${_parent_folder}/${_idl_stem}.json") - list(APPEND _generated_files - "${_output_path}/${_parent_folder}/${_idl_stem}.sha256") + set(_hash_file "${_output_path}/${_parent_folder}/${_idl_stem}.sha256") + list(APPEND _generated_hash_files ${_hash_file}) + list(APPEND _generated_hash_tuples "${_parent_folder}/${_idl_stem}:${_hash_file}") endforeach() set(_dependency_files "") @@ -43,7 +45,7 @@ foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) endforeach() endforeach() -set(${rosidl_generate_interfaces_TARGET}__JSON_IN_FILES ${_generated_json_in}) +set(${rosidl_generate_interfaces_TARGET}__HASH_TUPLES ${_generated_hash_tuples}) set(target_dependencies "${rosidl_generator_type_hash_BIN}" @@ -56,7 +58,7 @@ foreach(dep ${target_dependencies}) endif() endforeach() -set(_generated_files "${_generated_files};${_generated_json_in}") +set(_generated_files "${_generated_json};${_generated_json_in};${_generated_hash_files}") add_custom_command( COMMAND Python3::Interpreter ARGS diff --git a/rosidl_generator_type_hash/package.xml b/rosidl_generator_type_hash/package.xml index adb0dffcf..94293e985 100644 --- a/rosidl_generator_type_hash/package.xml +++ b/rosidl_generator_type_hash/package.xml @@ -16,12 +16,10 @@ ament_cmake_core python3 - rosidl_pycommon ament_index_python rosidl_cli rosidl_parser - rcutils ament_lint_auto ament_lint_common diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 8cd10dd85..14408d523 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ament_index_python import get_package_share_directory + from io import StringIO import json import os @@ -62,11 +64,28 @@ def generate_files( latest_target_timestamp = get_newest_modification_time(args['target_dependencies']) generated_files = [] + type_hash_files = {} + for hash_tuple in args.get('type_hash_tuples', []): + hash_parts = hash_tuple.split(':', 1) + assert len(hash_parts) == 2 + type_hash_files[hash_parts[0]] = hash_parts[1] + for idl_tuple in args.get('idl_tuples', []): idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 locator = IdlLocator(*idl_parts) idl_rel_path = pathlib.Path(idl_parts[1]) + + idl_rel_stem = idl_rel_path.with_suffix('') + try: + type_hash_file = type_hash_files[str(idl_rel_stem)] + with open(type_hash_file, 'r') as f: + type_hash_digest = f.read() + type_hash = bytes.fromhex(type_hash_digest) + except KeyError: + # TODO(emersonknapp) how to handle - typesupport generators don't need hash + type_hash = b'\0' * 32 + idl_stem = idl_rel_path.stem if not keep_case: idl_stem = convert_camel_case_to_lower_case_underscore(idl_stem) @@ -81,7 +100,7 @@ def generate_files( 'package_name': args['package_name'], 'interface_path': idl_rel_path, 'content': idl_file.content, - 'type_hash': b'\0'*32, + 'type_hash': type_hash, } if additional_context is not None: data.update(additional_context) From 572f13f1a4be1eb15481478b0d8dae3b3b2d66f5 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Feb 2023 15:50:48 -0800 Subject: [PATCH 15/65] Cleanup cmake comments Signed-off-by: Emerson Knapp --- ..._generator_type_hash_generate_interfaces.cmake | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index d4cab6970..a6d9329b5 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -88,20 +88,7 @@ if(NOT rosidl_generate_interfaces_SKIP_INSTALL) DESTINATION "share/${PROJECT_NAME}/${_parent_folder}") endforeach() - # Export old-style CMake variables - # ament_export_include_directories("include/${PROJECT_NAME}") - # ament_export_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix}) - + # TODO(emersonknapp) do I need this? # Export modern CMake targets # ament_export_targets(export_${rosidl_generate_interfaces_TARGET}${_target_suffix}) - # rosidl_export_typesupport_targets(${_target_suffix} - # ${rosidl_generate_interfaces_TARGET}${_target_suffix}) - - # install( - # TARGETS ${rosidl_generate_interfaces_TARGET}${_target_suffix} - # EXPORT export_${rosidl_generate_interfaces_TARGET}${_target_suffix} - # ARCHIVE DESTINATION lib - # LIBRARY DESTINATION lib - # RUNTIME DESTINATION bin - # ) endif() From 6f9620da92d122c58571ec2e8ba3bb0ef3c81e02 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Feb 2023 16:17:31 -0800 Subject: [PATCH 16/65] Use arguments json file instead of direct arguments Signed-off-by: Emerson Knapp --- .../rosidl_write_generator_arguments.cmake | 1 + .../bin/rosidl_generator_type_hash | 18 ++++-------------- ...nerator_type_hash_generate_interfaces.cmake | 14 ++++++++++---- .../rosidl_generator_type_hash/__init__.py | 14 ++++++++------ 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake b/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake index 5b5e03098..2c4d83cdf 100644 --- a/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake +++ b/rosidl_cmake/cmake/rosidl_write_generator_arguments.cmake @@ -33,6 +33,7 @@ function(rosidl_write_generator_arguments output_file) "ROS_INTERFACE_DEPENDENCIES" # since the dependencies can be empty "TARGET_DEPENDENCIES" "TYPE_HASH_TUPLES" + "INCLUDE_PATHS" "ADDITIONAL_FILES") cmake_parse_arguments( diff --git a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash index 63f4bb8e9..3bd8e5251 100755 --- a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash +++ b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash @@ -26,21 +26,11 @@ def main(argv=sys.argv[1:]): description='Generate hashable representations and hashes of ROS interfaces.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--package-name', required=True, help='TODO(emersonknapp') - parser.add_argument( - '--output-dir', required=True, help='TODO(emersonknapp') - parser.add_argument( - '--idl-tuples', nargs='+', required=True, help='TODO(emersonknapp)') - parser.add_argument( - '--include-paths', nargs='*', required=False, help='TODO(emersonknapp)') - + '--generator-arguments-file', + required=True, + help='The location of the file containing the generator arguments') args = parser.parse_args(argv) - - generate_type_hash( - package_name=args.package_name, - output_dir=args.output_dir, - idl_tuples=args.idl_tuples, - include_paths=args.include_paths) + generate_type_hash(args.generator_arguments_file) if __name__ == '__main__': diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index a6d9329b5..c41222c2c 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -58,15 +58,21 @@ foreach(dep ${target_dependencies}) endif() endforeach() +set(_generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash__arguments.json") +rosidl_write_generator_arguments( + "${_generator_arguments_file}" + PACKAGE_NAME "${PROJECT_NAME}" + IDL_TUPLES "${rosidl_generate_interfaces_IDL_TUPLES}" + OUTPUT_DIR "${_output_path}" + INCLUDE_PATHS "${_dependency_paths}" +) + set(_generated_files "${_generated_json};${_generated_json_in};${_generated_hash_files}") add_custom_command( COMMAND Python3::Interpreter ARGS ${rosidl_generator_type_hash_BIN} - --package-name "${PROJECT_NAME}" - --output-dir "${_output_path}" - --idl-tuples ${rosidl_generate_interfaces_IDL_TUPLES} - --include-paths ${_dependency_paths} + --generator-arguments-file "${_generator_arguments_file}" OUTPUT ${_generated_files} DEPENDS ${target_dependencies} COMMENT "Generating type hashes for ROS interfaces" diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index f0617e624..e4c9215a9 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -22,12 +22,14 @@ from rosidl_parser.parser import parse_idl_file -def generate_type_hash( - package_name: str, - output_dir: str, - idl_tuples: List[str], - include_paths: List[str] -): +def generate_type_hash(generator_arguments_file: str): + with open(generator_arguments_file, 'r') as f: + args = json.load(f) + package_name = args['package_name'] + output_dir = args['output_dir'] + idl_tuples = args['idl_tuples'] + include_paths = args.get('include_paths', []) + include_map = { package_name: Path(output_dir) } From fd8244565886cccb9e77967189e48ba1839de034 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 8 Feb 2023 14:08:55 -0800 Subject: [PATCH 17/65] Generate separate hashes and jsons for interface subtypes - not yet propagating to generated code Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 133 ++++++++++-------- 1 file changed, 75 insertions(+), 58 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index e4c9215a9..03efecd03 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -16,7 +16,7 @@ import json from pathlib import Path import sys -from typing import List +from typing import List, Tuple from rosidl_parser import definition from rosidl_parser.parser import parse_idl_file @@ -30,6 +30,7 @@ def generate_type_hash(generator_arguments_file: str): idl_tuples = args['idl_tuples'] include_paths = args.get('include_paths', []) + # Lookup for directory containing pregenerated .json.in files include_map = { package_name: Path(output_dir) } @@ -41,6 +42,8 @@ def generate_type_hash(generator_arguments_file: str): generated_files = [] json_contents = [] + + # First generate all .json.in files (so referenced types can be used in expansion) for idl_tuple in idl_tuples: idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 @@ -56,17 +59,18 @@ def generate_type_hash(generator_arguments_file: str): idl_stem = idl_rel_path.stem generate_to_dir = (Path(output_dir) / idl_rel_path).parent generate_to_dir.mkdir(parents=True, exist_ok=True) - stem_path = generate_to_dir / f'{idl_stem}' - json_in = generate_json_in(idl_file) - # Need to generate all .json.in before expanding .json final - json_contents.append((stem_path, json_in)) + # Generation will create several definitions for Services and Actions + for stem, contents in generate_json_in(idl_file, idl_stem): + stem_path = generate_to_dir / stem + json_contents.append((stem_path, contents)) - json_path = stem_path.with_suffix('.json.in') - with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json.dumps(json_in, indent=2)) - generated_files.append(str(json_path)) + json_path = stem_path.with_suffix('.json.in') + with json_path.open('w', encoding='utf-8') as json_file: + json_file.write(json.dumps(contents, indent=2)) + generated_files.append(str(json_path)) + # Expand .json.in and generate .sha256 hash files for stem_path, json_in in json_contents: json_out = generate_json_out(json_in, include_map) json_out_repr = json.dumps(json_out) @@ -74,11 +78,12 @@ def generate_type_hash(generator_arguments_file: str): json_path = stem_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: json_file.write(json_out_repr) - generated_files.append(str(json_path)) + generated_files.append(str(json_path)) sha = hashlib.sha256() sha.update(json_out_repr.encode('utf-8')) type_hash = sha.hexdigest() + hash_path = stem_path.with_suffix('.sha256') with hash_path.open('w', encoding='utf-8') as hash_file: hash_file.write(type_hash) @@ -88,7 +93,8 @@ def generate_type_hash(generator_arguments_file: str): # This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg -# TODO There is no FieldType.msg definition for the following rosidl_parser.definition types +# TODO(emersonknapp) +# There is no FieldType.msg definition for the following rosidl_parser.definition types # * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long # * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long FIELD_TYPES = { @@ -110,7 +116,8 @@ def generate_type_hash(generator_arguments_file: str): 'octet': 15, # byte definition.UnboundedString: 16, definition.UnboundedWString: 17, - # TODO there is no rosidl_parser.definition type for fixed strings (caveat: array of char) + # TODO(emersonknapp) + # there is no rosidl_parser.definition type FixedString # FIXED_STRING = 18 # FIXED_WSTRING = 19 definition.BoundedString: 20, @@ -119,28 +126,11 @@ def generate_type_hash(generator_arguments_file: str): NESTED_FIELD_TYPE_OFFSETS = { definition.Array: 32, - definition.BoundedSequence: 64, - definition.UnboundedSequence: 96, + definition.BoundedSequence: 32 * 2, + definition.UnboundedSequence: 32 * 3, } -def translate_type_id(value_type, offset, result): - if isinstance(value_type, definition.BasicType): - result['type_id'] = FIELD_TYPES[value_type.typename] + offset - elif isinstance(value_type, definition.AbstractGenericString): - result['type_id'] = FIELD_TYPES[type(value_type)] + offset - if value_type.has_maximum_size(): - result['string_length'] = value_type.maximum_size - elif isinstance(value_type, definition.NamespacedType): - result['type_id'] = offset - result['nested_type_name'] = '/'.join(value_type.namespaced_name()) - elif isinstance(value_type, definition.NamedType): - result['type_id'] = offset - result['nested_type_name'] = value_type.name - else: - raise TypeError('Unknown value type ', value_type) - - def serialize_field_type(ftype: definition.AbstractType): result = { 'type_id': 0, @@ -149,11 +139,12 @@ def serialize_field_type(ftype: definition.AbstractType): 'nested_type_name': '', } + type_id_offset = 0 if isinstance(ftype, definition.AbstractNestableType): - translate_type_id(ftype, 0, result) + value_type = ftype elif isinstance(ftype, definition.AbstractNestedType): type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] - translate_type_id(ftype.value_type, type_id_offset, result) + value_type = ftype.value_type if ftype.has_maximum_size(): try: result['length'] = ftype.maximum_size @@ -162,6 +153,21 @@ def serialize_field_type(ftype: definition.AbstractType): else: raise Exception('Unable to translate field type', ftype) + if isinstance(value_type, definition.BasicType): + result['type_id'] = FIELD_TYPES[value_type.typename] + type_id_offset + elif isinstance(value_type, definition.AbstractGenericString): + result['type_id'] = FIELD_TYPES[type(value_type)] + type_id_offset + if value_type.has_maximum_size(): + result['string_length'] = value_type.maximum_size + elif isinstance(value_type, definition.NamespacedType): + result['type_id'] = type_id_offset + result['nested_type_name'] = '/'.join(value_type.namespaced_name()) + elif isinstance(value_type, definition.NamedType): + result['type_id'] = type_id_offset + result['nested_type_name'] = value_type.name + else: + raise TypeError('Unknown value type ', value_type) + return result @@ -180,51 +186,62 @@ def serialize_individual_type_description(msg: definition.Message): } -def generate_json_in(idl: definition.IdlFile): - type_description = None +def generate_json_in(idl: definition.IdlFile, stem: str) -> List[Tuple[str, dict]]: + type_descriptions = [] includes = [] - # TODO(emersonknapp): do the individual message types get their own type hash? for el in idl.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) elif isinstance(el, definition.Message): - type_description = serialize_individual_type_description(el) + assert not type_descriptions, 'Found more than one interface in IDL' + type_descriptions = [ + (stem, serialize_individual_type_description(el)) + ] elif isinstance(el, definition.Service): - type_description = { - 'request_message': serialize_individual_type_description(el.request_message), - 'response_message': serialize_individual_type_description(el.response_message), + assert not type_descriptions, 'Found more than one interface in IDL' + request = serialize_individual_type_description(el.request_message) + response = serialize_individual_type_description(el.response_message) + service = { + 'request': request, + 'response': response, } + type_descriptions = [ + (stem, service), + (f'{stem}_Request', request), + (f'{stem}_Response', response), + ] elif isinstance(el, definition.Action): - type_description = { - 'send_goal_service': { - 'request_message': serialize_individual_type_description( - el.send_goal_service.request_message), - 'response_message': serialize_individual_type_description( - el.send_goal_service.response_message), - }, - 'get_result_service': { - 'request_message': serialize_individual_type_description( - el.get_result_service.request_message), - 'response_message': serialize_individual_type_description( - el.get_result_service.response_message), - }, - 'feedback_message': serialize_individual_type_description(el.feedback_message), + assert not type_descriptions, 'Found more than one interface in IDL' + goal = serialize_individual_type_description(el.goal) + result = serialize_individual_type_description(el.result) + feedback = serialize_individual_type_description(el.feedback) + action = { + 'goal': goal, + 'result': result, + 'feedback': feedback, } + type_descriptions = [ + (stem, action), + (f'{stem}_Goal', goal), + (f'{stem}_Result', result), + (f'{stem}_Feedback', feedback), + ] else: raise Exception(f'Do not know how to hash {el}') - if type_description is None: + if not type_descriptions: raise Exception('Did not find an interface to serialize in IDL file') includes = [ str(Path(include).with_suffix('.json.in')) for include in includes ] - return { + # TODO(emersonknapp) do I need to break parse which sub-interfaces use which includes? + return [(out_stem, { 'type_description': type_description, 'includes': includes, - } + }) for out_stem, type_description in type_descriptions] -def generate_json_out(json_in, includes_map): +def generate_json_out(json_in, includes_map) -> dict: pending_includes = json_in['includes'] loaded_includes = {} while pending_includes: From 048cc98f9f2a5ec184cfa6829bcf2c15f41296d2 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Feb 2023 16:33:16 -0800 Subject: [PATCH 18/65] Pretty print the type hashes in C/Cpp Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/msg__struct.h.em | 5 ++--- rosidl_generator_cpp/resource/msg__struct.hpp.em | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index 2a111c936..d62062bbd 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -62,10 +62,9 @@ for member in message.structure.members: @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> // Type Version Hash for interface -static const uint8_t @(idl_structure_type_to_c_typename(message.structure.namespaced_type))__TYPE_VERSION_HASH[32] = { - @ +static const uint8_t @(idl_structure_type_to_c_typename(message.structure.namespaced_type))__TYPE_VERSION_HASH[32] = {@ @[for i in range(32)]@ -0x@(type_hash[i:i+1].hex()), @ +@[if i % 8 == 0]@\n @[end if]0x@(type_hash[i:i+1].hex()), @ @[end for] }; diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index 2db39d23c..d10c6e092 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -103,10 +103,9 @@ struct @(message.structure.namespaced_type.name)_ // Type Version Hash for interface - static constexpr uint8_t TYPE_VERSION_HASH[32] = { - @ -@[for i in range(32)]@ -0x@(type_hash[i:i+1].hex()), @ + static constexpr uint8_t TYPE_VERSION_HASH[32] = {@ + @[for i in range(32)]@ +@[if i % 8 == 0]@\n @[end if]0x@(type_hash[i:i+1].hex()), @ @[end for] }; From 9d4636a5a10499848db2fd96d4645d932b8a4339 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Feb 2023 16:39:46 -0800 Subject: [PATCH 19/65] Code cleanup pass Signed-off-by: Emerson Knapp --- rosidl_generator_type_hash/CHANGELOG.rst | 141 ------------------ rosidl_generator_type_hash/package.xml | 2 +- ...rosidl_generator_type_hash-extras.cmake.in | 5 - .../rosidl_generator_type_hash/__init__.py | 2 +- rosidl_pycommon/rosidl_pycommon/__init__.py | 2 - 5 files changed, 2 insertions(+), 150 deletions(-) delete mode 100644 rosidl_generator_type_hash/CHANGELOG.rst diff --git a/rosidl_generator_type_hash/CHANGELOG.rst b/rosidl_generator_type_hash/CHANGELOG.rst deleted file mode 100644 index dafdc2ab2..000000000 --- a/rosidl_generator_type_hash/CHANGELOG.rst +++ /dev/null @@ -1,141 +0,0 @@ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Changelog for package rosidl_generator_c -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -3.3.1 (2022-11-02) ------------------- - -3.3.0 (2022-09-08) ------------------- -* Move rosidl_generator_c/cpp tests to a separate package (`#701 `_) -* Move rosidl_cmake Python module to a new package rosidl_pycommon (`#696 `_) - Deprecate the Python module in rosidl_cmake and move the implementation to the new package rosidl_pycommon. -* Add namespaced ALIAS target to easily consume generated libraries via add_subdirectory (`#605 `_) -* Contributors: Jacob Perron, Silvio Traversaro - -3.2.1 (2022-06-21) ------------------- - -3.2.0 (2022-05-04) ------------------- - -3.1.3 (2022-04-08) ------------------- - -3.1.2 (2022-04-05) ------------------- -* Fix error handling when copying C sequence messages (`#671 `_) -* Contributors: Michel Hidalgo - -3.1.1 (2022-03-28) ------------------- -* Install generated headers to include/${PROJECT_NAME} (`#670 `_) -* Misc cleanup in the rosidl generator extensions (`#662 `_) -* Set the output size unconditionally when copying sequences (`#669 `_) -* Contributors: Nikolai Morin, Shane Loretz - -3.1.0 (2022-03-01) ------------------- -* Implement copy function for C messages (`#650 `_) -* Implement equality operator function for C messages. (`#648 `_) -* Generate documentation in generated C header files based on ROS interfaces comments (`#593 `_) -* Contributors: Ivan Santiago Paunovic, Michel Hidalgo - -3.0.1 (2022-01-13) ------------------- - -3.0.0 (2021-11-05) ------------------- -* Update package maintainers (`#624 `_) -* Make rosidl packages use FindPython3 instead of FindPythonInterp (`#612 `_) -* Contributors: Michel Hidalgo, Shane Loretz - -2.5.0 (2021-08-10) ------------------- -* Revert "Bundle and ensure the exportation of rosidl generated targets" (`#611 `_) -* Contributors: Michel Hidalgo - -2.4.0 (2021-07-12) ------------------- -* Bundle and ensure the exportation of rosidl generated targets (`#601 `_) -* Contributors: Michel Hidalgo - -2.3.0 (2021-06-11) ------------------- -* Fix a cpplint allocator regression. (`#590 `_) -* Use RCUtils allocators in rosidl_generator_c (`#584 `_) -* Contributors: Chris Lalancette, Pablo Garrido - -2.2.1 (2021-04-06) ------------------- - -2.2.0 (2021-03-18) ------------------- -* Expose C code generation via rosidl generate CLI (`#569 `_) -* Contributors: Michel Hidalgo - -2.1.0 (2021-03-09) ------------------- - -2.0.3 (2021-01-25) ------------------- - -2.0.2 (2020-12-08) ------------------- -* Strip action service suffixes from C include prefix (`#538 `_) -* Update the maintainers of this repository. (`#536 `_) -* Contributors: Chris Lalancette, Jacob Perron - -2.0.1 (2020-09-28) ------------------- - -2.0.0 (2020-09-24) ------------------- -* Fix the declared language for a few packages (`#530 `_) -* Contributors: Scott K Logan - -1.1.0 (2020-08-17) ------------------- -* Do not depend on rosidl_runtime_c when tests are disabled (`#503 `_) -* Contributors: Ben Wolsieffer - -1.0.1 (2020-06-03) ------------------- - -1.0.0 (2020-05-22) ------------------- - -0.9.2 (2020-05-19) ------------------- - -0.9.1 (2020-05-08) ------------------- - -0.9.0 (2020-04-24) ------------------- -* Export targets in addition to include directories / libraries (`#473 `_) -* Move non-entry point headers into detail subdirectory (`#461 `_) -* Rename rosidl_generator_c 'namespace' to rosidl_runtime_c (`#458 `_) -* Only export ament_cmake_core instead of ament_cmake (`#459 `_) -* Split rosidl_generator_c and rosidl_generator_cpp in two: rosidl_generator_x and rosidl_runtime_x (`#442 `_) -* Added rosidl_generator_c as a member of group rosidl_runtime_packages (`#440 `_) -* Style update to match uncrustify with explicit language (`#439 `_) -* Code style only: wrap after open parenthesis if not in one line (`#435 `_) -* Use f-string (`#436 `_) -* Move repeated logic for C include prefix into common function (`#432 `_) -* Contributors: Alejandro Hernández Cordero, Dirk Thomas, Jacob Perron - -0.8.2 (2020-01-17) ------------------- -* Fix double free issue when initialization is failed (`#423 `_) -* Contributors: DongheeYe - -0.8.1 (2019-10-23) ------------------- - -0.8.0 (2019-09-24) ------------------- -* [rosidl_generator_c] Updated tests for new msg types from test_interface_files (`#398 `_) -* use latin-1 encoding when reading/writing .idl files, prepend BOM to generated C/C++ files when necessary (`#391 `_) -* Set _FOUND to trick ament_target_dependencies() for test (`#396 `_) -* Contributors: Dirk Thomas, Shane Loretz, Siddharth Kucheria diff --git a/rosidl_generator_type_hash/package.xml b/rosidl_generator_type_hash/package.xml index 94293e985..c00156a87 100644 --- a/rosidl_generator_type_hash/package.xml +++ b/rosidl_generator_type_hash/package.xml @@ -3,7 +3,7 @@ rosidl_generator_type_hash 3.3.1 - Generate the ROS interfaces in C. + Generate REP-2011 type version hashes for ROS interfaces. Emerson Knapp diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in index 95b4084a0..2dc1d2e96 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in @@ -14,8 +14,3 @@ set(rosidl_generator_type_hash_GENERATOR_FILES "${rosidl_generator_type_hash_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_hash/__init__.py") normalize_path(rosidl_generator_type_hash_GENERATOR_FILES "${rosidl_generator_type_hash_GENERATOR_FILES}") - -set(rosidl_generator_type_hash_TEMPLATE_DIR - "${rosidl_generator_type_hash_DIR}/../resource") -normalize_path(rosidl_generator_type_hash_TEMPLATE_DIR - "${rosidl_generator_type_hash_TEMPLATE_DIR}") diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 03efecd03..6b986f3b3 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2015 Open Source Robotics Foundation, Inc. +# Copyright 2023 Open Source Robotics Foundation, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 14408d523..9055136ba 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ament_index_python import get_package_share_directory - from io import StringIO import json import os From c5b1364f60a328a5b862adbae2bd498a68148b9b Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 11:10:47 -0800 Subject: [PATCH 20/65] WIP defining type hash structure i need by usage patterns Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 39 ++++++++---- .../resource/action__struct.hpp.em | 18 ++++-- .../resource/srv__struct.hpp.em | 9 ++- .../rosidl_generator_type_hash/__init__.py | 62 +++++++++---------- rosidl_pycommon/rosidl_pycommon/__init__.py | 8 +-- 5 files changed, 79 insertions(+), 57 deletions(-) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index 304a71aaa..42a63d277 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -59,21 +59,24 @@ from rosidl_parser.definition import Service TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=service.request_message, include_directives=include_directives, type_hash=type_hash) + message=service.request_message, include_directives=include_directives, + type_hash=type_hash['request_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=service.response_message, include_directives=include_directives, type_hash=type_hash) + message=service.response_message, include_directives=include_directives, + type_hash=type_hash['response_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=service.event_message, include_directives=include_directives, type_hash=type_hash) + message=service.event_message, include_directives=include_directives, + type_hash=type_hash['event_message']) }@ @[end for]@ @@ -89,70 +92,80 @@ from rosidl_parser.definition import Action TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.goal, include_directives=include_directives, type_hash=type_hash) + message=action.goal, include_directives=include_directives, + type_hash=type_hash['goal']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.result, include_directives=include_directives, type_hash=type_hash) + message=action.result, include_directives=include_directives, + type_hash=type_hash['result']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.feedback, include_directives=include_directives, type_hash=type_hash) + message=action.feedback, include_directives=include_directives, + type_hash=type_hash['feedback']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.send_goal_service.request_message, include_directives=include_directives, type_hash=type_hash) + message=action.send_goal_service.request_message, include_directives=include_directives, + type_hash=type_hash['send_goal_service']['request_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.send_goal_service.response_message, include_directives=include_directives, type_hash=type_hash) + message=action.send_goal_service.response_message, include_directives=include_directives, + type_hash=type_hash['send_goal_service']['response_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.send_goal_service.event_message, include_directives=include_directives, type_hash=type_hash) + message=action.send_goal_service.event_message, include_directives=include_directives, + type_hash=type_hash['send_goal_service']['event_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.get_result_service.request_message, include_directives=include_directives, type_hash=type_hash) + message=action.get_result_service.request_message, include_directives=include_directives, + type_hash=type_hash['get_result_service']['request_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.get_result_service.response_message, include_directives=include_directives, type_hash=type_hash) + message=action.get_result_service.response_message, include_directives=include_directives, + type_hash=type_hash['get_result_service']['response_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.get_result_service.event_message, include_directives=include_directives, type_hash=type_hash) + message=action.get_result_service.event_message, include_directives=include_directives, + type_hash=type_hash['get_result_service']['event_message']) }@ @{ TEMPLATE( 'msg__struct.h.em', package_name=package_name, interface_path=interface_path, - message=action.feedback_message, include_directives=include_directives, type_hash=type_hash) + message=action.feedback_message, include_directives=include_directives, + type_hash=type_hash['feedback_message']) }@ @[end for]@ diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index 6d1a3f12e..f6527f85f 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -17,42 +17,48 @@ action_name = '::'.join(action.namespaced_type.namespaced_name()) TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.goal, include_directives=include_directives, type_hash=type_hash) + message=action.goal, include_directives=include_directives, + type_hash=type_hash['goal']) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.result, include_directives=include_directives, type_hash=type_hash) + message=action.result, include_directives=include_directives, + type_hash=type_hash['result']) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.feedback, include_directives=include_directives, type_hash=type_hash) + message=action.feedback, include_directives=include_directives, + type_hash=type_hash['feedback']) }@ @{ TEMPLATE( 'srv__struct.hpp.em', package_name=package_name, interface_path=interface_path, - service=action.send_goal_service, include_directives=include_directives, type_hash=type_hash) + service=action.send_goal_service, include_directives=include_directives, + type_hash=type_hash['send_goal_service']) }@ @{ TEMPLATE( 'srv__struct.hpp.em', package_name=package_name, interface_path=interface_path, - service=action.get_result_service, include_directives=include_directives, type_hash=type_hash) + service=action.get_result_service, include_directives=include_directives, + type_hash=type_hash['get_result_service']) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=action.feedback_message, include_directives=include_directives, type_hash=type_hash) + message=action.feedback_message, include_directives=include_directives, + type_hash=type_hash['feedback_message']) }@ @[for header_file in action_includes]@ diff --git a/rosidl_generator_cpp/resource/srv__struct.hpp.em b/rosidl_generator_cpp/resource/srv__struct.hpp.em index c18a11547..4c870323e 100644 --- a/rosidl_generator_cpp/resource/srv__struct.hpp.em +++ b/rosidl_generator_cpp/resource/srv__struct.hpp.em @@ -3,21 +3,24 @@ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=service.request_message, include_directives=include_directives, type_hash=type_hash) + message=service.request_message, include_directives=include_directives, + type_hash=type_hash['request_message']) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=service.response_message, include_directives=include_directives, type_hash=type_hash) + message=service.response_message, include_directives=include_directives, + type_hash=type_hash['response_message']) }@ @{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, - message=service.event_message, include_directives=include_directives, type_hash=type_hash) + message=service.event_message, include_directives=include_directives, + type_hash=type_hash['event_message']) }@ @[for ns in service.namespaced_type.namespaces]@ diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 6b986f3b3..492796a5d 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -187,58 +187,58 @@ def serialize_individual_type_description(msg: definition.Message): def generate_json_in(idl: definition.IdlFile, stem: str) -> List[Tuple[str, dict]]: - type_descriptions = [] + type_description = None includes = [] for el in idl.content.elements: if isinstance(el, definition.Include): includes.append(el.locator) elif isinstance(el, definition.Message): - assert not type_descriptions, 'Found more than one interface in IDL' - type_descriptions = [ + assert not type_description, 'Found more than one interface in IDL' + type_description = [ (stem, serialize_individual_type_description(el)) ] elif isinstance(el, definition.Service): - assert not type_descriptions, 'Found more than one interface in IDL' - request = serialize_individual_type_description(el.request_message) - response = serialize_individual_type_description(el.response_message) + assert not type_description, 'Found more than one interface in IDL' + request_message = serialize_individual_type_description(el.request_message) + response_message = serialize_individual_type_description(el.response_message) service = { - 'request': request, - 'response': response, + 'request_message': request_message, + 'response_message': response_message, } - type_descriptions = [ - (stem, service), - (f'{stem}_Request', request), - (f'{stem}_Response', response), - ] - elif isinstance(el, definition.Action): - assert not type_descriptions, 'Found more than one interface in IDL' - goal = serialize_individual_type_description(el.goal) - result = serialize_individual_type_description(el.result) - feedback = serialize_individual_type_description(el.feedback) - action = { - 'goal': goal, - 'result': result, - 'feedback': feedback, + type_description = { + 'service': service, + 'request_message': request_message, + 'response_message': response_message, } - type_descriptions = [ - (stem, action), - (f'{stem}_Goal', goal), - (f'{stem}_Result', result), - (f'{stem}_Feedback', feedback), - ] + elif isinstance(el, definition.Action): + assert not type_description, 'Found more than one interface in IDL' + # goal = serialize_individual_type_description(el.goal) + # result = serialize_individual_type_description(el.result) + # feedback = serialize_individual_type_description(el.feedback) + # action = { + # 'goal': goal, + # 'result': result, + # 'feedback': feedback, + # } + # type_descriptions = [ + # (stem, action), + # (f'{stem}_Goal', goal), + # (f'{stem}_Result', result), + # (f'{stem}_Feedback', feedback), + # ] else: raise Exception(f'Do not know how to hash {el}') - if not type_descriptions: + if not type_description: raise Exception('Did not find an interface to serialize in IDL file') includes = [ str(Path(include).with_suffix('.json.in')) for include in includes ] # TODO(emersonknapp) do I need to break parse which sub-interfaces use which includes? - return [(out_stem, { + return { 'type_description': type_description, 'includes': includes, - }) for out_stem, type_description in type_descriptions] + } def generate_json_out(json_in, includes_map) -> dict: diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 9055136ba..b19eebdef 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -78,11 +78,11 @@ def generate_files( try: type_hash_file = type_hash_files[str(idl_rel_stem)] with open(type_hash_file, 'r') as f: - type_hash_digest = f.read() - type_hash = bytes.fromhex(type_hash_digest) + type_hash_infos = f.read() + # type_hash = bytes.fromhex(type_hash_digest) except KeyError: # TODO(emersonknapp) how to handle - typesupport generators don't need hash - type_hash = b'\0' * 32 + type_hash_infos = None idl_stem = idl_rel_path.stem if not keep_case: @@ -98,7 +98,7 @@ def generate_files( 'package_name': args['package_name'], 'interface_path': idl_rel_path, 'content': idl_file.content, - 'type_hash': type_hash, + 'type_hash': type_hash_infos, } if additional_context is not None: data.update(additional_context) From 713132b439616e25d58e6610eb1e475c7c1b85a5 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 15:20:31 -0800 Subject: [PATCH 21/65] New nested version working for msg/srv Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 3 + rosidl_generator_c/resource/msg__struct.h.em | 6 +- .../resource/msg__struct.hpp.em | 7 +- .../resource/srv__struct.hpp.em | 2 + .../rosidl_generator_type_hash/__init__.py | 236 ++++++++++-------- rosidl_pycommon/rosidl_pycommon/__init__.py | 21 +- 6 files changed, 157 insertions(+), 118 deletions(-) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index 42a63d277..fc289a729 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -11,6 +11,7 @@ @# - content (IdlContent, list of elements, e.g. Messages or Services) @####################################################################### @{ +from rosidl_generator_c import idl_structure_type_to_c_typename from rosidl_pycommon import convert_camel_case_to_lower_case_underscore include_parts = [package_name] + list(interface_path.parents[0].parts) + [ 'detail', convert_camel_case_to_lower_case_underscore(interface_path.stem)] @@ -55,6 +56,8 @@ TEMPLATE( from rosidl_parser.definition import Service }@ @[for service in content.get_elements_of_type(Service)]@ +static const @(TYPE_HASH(idl_structure_type_to_c_typename(service.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['service']))@ + @{ TEMPLATE( 'msg__struct.h.em', diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index d62062bbd..5d41ca389 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -62,11 +62,7 @@ for member in message.structure.members: @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> // Type Version Hash for interface -static const uint8_t @(idl_structure_type_to_c_typename(message.structure.namespaced_type))__TYPE_VERSION_HASH[32] = {@ -@[for i in range(32)]@ -@[if i % 8 == 0]@\n @[end if]0x@(type_hash[i:i+1].hex()), @ -@[end for] -}; +static const @(TYPE_HASH(idl_structure_type_to_c_typename(message.structure.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['message']))@ @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< // Constants defined in the message diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index d10c6e092..0f54930fb 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -101,13 +101,8 @@ struct @(message.structure.namespaced_type.name)_ { using Type = @(message.structure.namespaced_type.name)_; - // Type Version Hash for interface - static constexpr uint8_t TYPE_VERSION_HASH[32] = {@ - @[for i in range(32)]@ -@[if i % 8 == 0]@\n @[end if]0x@(type_hash[i:i+1].hex()), @ -@[end for] - }; + static constexpr @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['message'], indent=2))@ @{ # The creation of the constructors for messages is a bit complicated. The goal diff --git a/rosidl_generator_cpp/resource/srv__struct.hpp.em b/rosidl_generator_cpp/resource/srv__struct.hpp.em index 4c870323e..3b0cd166c 100644 --- a/rosidl_generator_cpp/resource/srv__struct.hpp.em +++ b/rosidl_generator_cpp/resource/srv__struct.hpp.em @@ -34,6 +34,8 @@ struct @(service.namespaced_type.name) @{ service_typename = '::'.join(service.namespaced_type.namespaced_name()) }@ + static constexpr @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['service'], indent=2))@ + using Request = @(service_typename)_Request; using Response = @(service_typename)_Response; using Event = @(service_typename)_Event; diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 492796a5d..16c7a16cb 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -26,13 +26,13 @@ def generate_type_hash(generator_arguments_file: str): with open(generator_arguments_file, 'r') as f: args = json.load(f) package_name = args['package_name'] - output_dir = args['output_dir'] + output_dir = Path(args['output_dir']) idl_tuples = args['idl_tuples'] include_paths = args.get('include_paths', []) # Lookup for directory containing pregenerated .json.in files include_map = { - package_name: Path(output_dir) + package_name: output_dir } for include_tuple in include_paths: include_parts = include_tuple.rsplit(':', 1) @@ -41,7 +41,7 @@ def generate_type_hash(generator_arguments_file: str): include_map[include_package_name] = Path(include_base_path) generated_files = [] - json_contents = [] + hashers = [] # First generate all .json.in files (so referenced types can be used in expansion) for idl_tuple in idl_tuples: @@ -56,38 +56,21 @@ def generate_type_hash(generator_arguments_file: str): raise(e) idl_rel_path = Path(idl_parts[1]) - idl_stem = idl_rel_path.stem - generate_to_dir = (Path(output_dir) / idl_rel_path).parent + print(idl_rel_path) + generate_to_dir = (output_dir / idl_rel_path).parent generate_to_dir.mkdir(parents=True, exist_ok=True) - # Generation will create several definitions for Services and Actions - for stem, contents in generate_json_in(idl_file, idl_stem): - stem_path = generate_to_dir / stem - json_contents.append((stem_path, contents)) - - json_path = stem_path.with_suffix('.json.in') - with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json.dumps(contents, indent=2)) - generated_files.append(str(json_path)) + hasher = InterfaceHasher.from_idl(idl_file, idl_rel_path) + generated_files.extend( + hasher.write_json_in(output_dir)) + hashers.append(hasher) # Expand .json.in and generate .sha256 hash files - for stem_path, json_in in json_contents: - json_out = generate_json_out(json_in, include_map) - json_out_repr = json.dumps(json_out) - - json_path = stem_path.with_suffix('.json') - with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json_out_repr) - generated_files.append(str(json_path)) - - sha = hashlib.sha256() - sha.update(json_out_repr.encode('utf-8')) - type_hash = sha.hexdigest() - - hash_path = stem_path.with_suffix('.sha256') - with hash_path.open('w', encoding='utf-8') as hash_file: - hash_file.write(type_hash) - generated_files.append(str(hash_path)) + for hasher in hashers: + generated_files.extend( + hasher.write_json_out(output_dir, include_map)) + generated_files.extend( + hasher.write_hash(output_dir)) return generated_files @@ -186,81 +169,130 @@ def serialize_individual_type_description(msg: definition.Message): } -def generate_json_in(idl: definition.IdlFile, stem: str) -> List[Tuple[str, dict]]: - type_description = None - includes = [] - for el in idl.content.elements: - if isinstance(el, definition.Include): - includes.append(el.locator) - elif isinstance(el, definition.Message): - assert not type_description, 'Found more than one interface in IDL' - type_description = [ - (stem, serialize_individual_type_description(el)) - ] - elif isinstance(el, definition.Service): - assert not type_description, 'Found more than one interface in IDL' - request_message = serialize_individual_type_description(el.request_message) - response_message = serialize_individual_type_description(el.response_message) - service = { - 'request_message': request_message, - 'response_message': response_message, - } - type_description = { - 'service': service, - 'request_message': request_message, - 'response_message': response_message, - } - elif isinstance(el, definition.Action): - assert not type_description, 'Found more than one interface in IDL' - # goal = serialize_individual_type_description(el.goal) - # result = serialize_individual_type_description(el.result) - # feedback = serialize_individual_type_description(el.feedback) - # action = { - # 'goal': goal, - # 'result': result, - # 'feedback': feedback, - # } - # type_descriptions = [ - # (stem, action), - # (f'{stem}_Goal', goal), - # (f'{stem}_Result', result), - # (f'{stem}_Feedback', feedback), - # ] - else: - raise Exception(f'Do not know how to hash {el}') - if not type_description: - raise Exception('Did not find an interface to serialize in IDL file') - - includes = [ - str(Path(include).with_suffix('.json.in')) for include in includes - ] - # TODO(emersonknapp) do I need to break parse which sub-interfaces use which includes? +def serialize_individual_service_description(srv: definition.Service): + request_type = definition.NamespacedType( + srv.namespaced_type.namespaces, f'{srv.namespaced_type.name}_Request') + response_type = definition.NamespacedType( + srv.namespaced_type.namespaces, f'{srv.namespaced_type.name}_Response') return { - 'type_description': type_description, - 'includes': includes, + 'type_name': '/'.join(srv.namespaced_type.namespaced_name()), + 'fields': [ + serialize_field(definition.Member(request_type, 'request_message')), + serialize_field(definition.Member(response_type, 'response_message')), + ] } -def generate_json_out(json_in, includes_map) -> dict: - pending_includes = json_in['includes'] - loaded_includes = {} - while pending_includes: - process_include = pending_includes.pop() - if process_include in loaded_includes: - continue - p_path = Path(process_include) - assert(not p_path.is_absolute()) - include_package = p_path.parts[0] - include_file = includes_map[include_package] / p_path.relative_to(include_package) +def serialize_individual_action_description(action: definition.Action): + raise Exception('Action plz') + + +class InterfaceHasher: + + @classmethod + def from_idl(cls, idl: definition.IdlFile, idl_rel_path: str): + includes = idl.content.get_elements_of_type(definition.Include) + for el in idl.content.elements: + if isinstance(el, definition.Message): + return InterfaceHasher(el, includes, idl_rel_path) + elif isinstance(el, definition.Service): + return InterfaceHasher(el, includes, idl_rel_path) + elif isinstance(el, definition.Action): + return InterfaceHasher(el, includes, idl_rel_path) + raise Exception('No interface found in IDL') + + def __init__(self, interface, includes, idl_rel_path: str): + self.includes = [str(Path(include).with_suffix('.json.in')) for include in includes] + self.interface = interface + self.interface_type = '' + self.rel_path = idl_rel_path + self.subinterfaces = {} + + if isinstance(interface, definition.Message): + self.interface_type = 'message' + self.individual_type_description = serialize_individual_type_description(interface) + elif isinstance(interface, definition.Service): + self.interface_type = 'service' + stem = idl_rel_path.stem + self.subinterfaces = { + 'request_message': InterfaceHasher( + interface.request_message, includes, + idl_rel_path.parent / f'{stem}_Request'), + 'response_message': InterfaceHasher( + interface.response_message, includes, + idl_rel_path.parent / f'{stem}_Response'), + 'event_message': InterfaceHasher( + interface.event_message, includes, + idl_rel_path.parent / f'{stem}_Event'), + } + self.individual_type_description = serialize_individual_service_description(interface) + elif isinstance(interface, definition.Action): + raise Exception('Action plz') + + self.json_in = { + 'type_description': self.individual_type_description, + 'includes': self.includes, + } - with include_file.open('r') as include_file: - include_json = json.load(include_file) + def write_json_in(self, output_dir): + for key, val in self.subinterfaces.items(): + val.write_json_in(output_dir) - loaded_includes[process_include] = include_json['type_description'] - pending_includes.extend(include_json['includes']) + json_path = (output_dir / self.rel_path).with_suffix('.json.in') + json_path.parent.mkdir(parents=True, exist_ok=True) + with json_path.open('w', encoding='utf-8') as json_file: + json_file.write(json.dumps(self.json_in, indent=2)) + return [str(json_path)] + + def write_json_out(self, output_dir, includes_map): + for key, val in self.subinterfaces.items(): + val.write_json_out(output_dir, includes_map) + + pending_includes = self.json_in['includes'] + loaded_includes = {} + while pending_includes: + process_include = pending_includes.pop() + if process_include in loaded_includes: + continue + p_path = Path(process_include) + assert(not p_path.is_absolute()) + include_package = p_path.parts[0] + include_file = includes_map[include_package] / p_path.relative_to(include_package) + + with include_file.open('r') as include_file: + include_json = json.load(include_file) + + loaded_includes[process_include] = include_json['type_description'] + pending_includes.extend(include_json['includes']) + + self.json_out = { + 'type_description': self.json_in['type_description'], + 'referenced_type_descriptions': sorted( + loaded_includes.values(), key=lambda td: td['type_name']) + } + + json_path = (output_dir / self.rel_path).with_suffix('.json') + with json_path.open('w', encoding='utf-8') as json_file: + json_file.write(json.dumps(self.json_out, indent=2)) + return [str(json_path)] - return { - 'type_description': json_in['type_description'], - 'referenced_type_descriptions': sorted( - loaded_includes.values(), key=lambda td: td['type_name']) - } + def calculate_hash(self): + json_out_repr = json.dumps(self.json_out) + sha = hashlib.sha256() + sha.update(json_out_repr.encode('utf-8')) + type_hash = sha.hexdigest() + + type_hash_infos = { + self.interface_type: type_hash, + } + for key, val in self.subinterfaces.items(): + type_hash_infos[key] = val.calculate_hash() + + return type_hash_infos + + def write_hash(self, output_dir): + type_hash = self.calculate_hash() + hash_path = (output_dir / self.rel_path).with_suffix('.sha256') + with hash_path.open('w', encoding='utf-8') as hash_file: + hash_file.write(json.dumps(type_hash, indent=2)) + return [str(hash_path)] diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index b19eebdef..ff4abaf37 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -62,6 +62,7 @@ def generate_files( latest_target_timestamp = get_newest_modification_time(args['target_dependencies']) generated_files = [] + type_hashes_provided = 'type_hash_tuples' in args type_hash_files = {} for hash_tuple in args.get('type_hash_tuples', []): hash_parts = hash_tuple.split(':', 1) @@ -75,13 +76,11 @@ def generate_files( idl_rel_path = pathlib.Path(idl_parts[1]) idl_rel_stem = idl_rel_path.with_suffix('') - try: + if type_hashes_provided: type_hash_file = type_hash_files[str(idl_rel_stem)] with open(type_hash_file, 'r') as f: - type_hash_infos = f.read() - # type_hash = bytes.fromhex(type_hash_digest) - except KeyError: - # TODO(emersonknapp) how to handle - typesupport generators don't need hash + type_hash_infos = json.load(f) + else: type_hash_infos = None idl_stem = idl_rel_path.stem @@ -202,6 +201,7 @@ def expand_template( def _add_helper_functions(data): data['TEMPLATE'] = _expand_template + data['TYPE_HASH'] = _expand_type_hash def _expand_template(template_name, **kwargs): @@ -219,3 +219,14 @@ def _expand_template(template_name, **kwargs): file=sys.stderr) raise interpreter.invoke('afterInclude') + + +def _expand_type_hash(variable_name, hex_string, indent=0): + ind_str = ' ' * indent + result = f'uint8_t {variable_name}[32] = {{' + for i in range(32): + if i % 8 == 0: + result += f'\n{ind_str} ' + result += f'0x{hex_string[i*2:i*2+1]}, ' + result += f'\n{ind_str}}};\n' + return result From ce87154b720e9039e46b5e4f3fda5f227b66bcf9 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 15:48:33 -0800 Subject: [PATCH 22/65] Add action breakdown Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 92 +++++++++++++++---- 1 file changed, 76 insertions(+), 16 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 16c7a16cb..e65ff694c 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -56,7 +56,6 @@ def generate_type_hash(generator_arguments_file: str): raise(e) idl_rel_path = Path(idl_parts[1]) - print(idl_rel_path) generate_to_dir = (output_dir / idl_rel_path).parent generate_to_dir.mkdir(parents=True, exist_ok=True) @@ -170,21 +169,59 @@ def serialize_individual_type_description(msg: definition.Message): def serialize_individual_service_description(srv: definition.Service): - request_type = definition.NamespacedType( - srv.namespaced_type.namespaces, f'{srv.namespaced_type.name}_Request') - response_type = definition.NamespacedType( - srv.namespaced_type.namespaces, f'{srv.namespaced_type.name}_Response') + name = srv.namespaced_type.name + members = [ + definition.Member( + definition.NamespacedType( + srv.namespaced_type.namespaces, + name + definition.SERVICE_REQUEST_MESSAGE_SUFFIX), 'request_message'), + definition.Member( + definition.NamespacedType( + srv.namespaced_type.namespaces, + name + definition.SERVICE_RESPONSE_MESSAGE_SUFFIX), 'response_message'), + definition.Member( + definition.NamespacedType( + srv.namespaced_type.namespaces, + name + definition.SERVICE_EVENT_MESSAGE_SUFFIX), 'event_message'), + ] return { 'type_name': '/'.join(srv.namespaced_type.namespaced_name()), - 'fields': [ - serialize_field(definition.Member(request_type, 'request_message')), - serialize_field(definition.Member(response_type, 'response_message')), - ] + 'fields': [serialize_field(member) for member in members], } def serialize_individual_action_description(action: definition.Action): - raise Exception('Action plz') + name = action.namespaced_type.name + members = [ + definition.Member( + definition.NamespacedType( + action.namespaced_type.namespaces, + name + definition.ACTION_GOAL_SUFFIX), 'goal'), + definition.Member( + definition.NamespacedType( + action.namespaced_type.namespaces, + name + definition.ACTION_RESULT_SUFFIX), 'result'), + definition.Member( + definition.NamespacedType( + action.namespaced_type.namespaces, + name + definition.ACTION_FEEDBACK_SUFFIX), 'feedback'), + definition.Member( + definition.NamespacedType( + action.namespaced_type.namespaces, + name + definition.ACTION_GOAL_SERVICE_SUFFIX), 'send_goal_service'), + definition.Member( + definition.NamespacedType( + action.namespaced_type.namespaces, + name + definition.ACTION_RESULT_SERVICE_SUFFIX), 'get_result_service'), + definition.Member( + definition.NamespacedType( + action.namespaced_type.namespaces, + name + definition.ACTION_FEEDBACK_MESSAGE_SUFFIX), 'feedback_message'), + ] + return { + 'type_name': '/'.join(action.namespaced_type.namespaced_name()), + 'fields': [serialize_field(member) for member in members], + } class InterfaceHasher: @@ -202,32 +239,55 @@ def from_idl(cls, idl: definition.IdlFile, idl_rel_path: str): raise Exception('No interface found in IDL') def __init__(self, interface, includes, idl_rel_path: str): - self.includes = [str(Path(include).with_suffix('.json.in')) for include in includes] + self.includes = [str(Path(include.locator).with_suffix('.json.in')) for include in includes] self.interface = interface self.interface_type = '' self.rel_path = idl_rel_path self.subinterfaces = {} + def subipath(suffix): + return idl_rel_path.parent / (idl_rel_path.stem + suffix) + if isinstance(interface, definition.Message): self.interface_type = 'message' self.individual_type_description = serialize_individual_type_description(interface) elif isinstance(interface, definition.Service): self.interface_type = 'service' - stem = idl_rel_path.stem self.subinterfaces = { 'request_message': InterfaceHasher( interface.request_message, includes, - idl_rel_path.parent / f'{stem}_Request'), + subipath(definition.SERVICE_REQUEST_MESSAGE_SUFFIX)), 'response_message': InterfaceHasher( interface.response_message, includes, - idl_rel_path.parent / f'{stem}_Response'), + subipath(definition.SERVICE_RESPONSE_MESSAGE_SUFFIX)), 'event_message': InterfaceHasher( interface.event_message, includes, - idl_rel_path.parent / f'{stem}_Event'), + subipath(definition.SERVICE_EVENT_MESSAGE_SUFFIX)), } self.individual_type_description = serialize_individual_service_description(interface) elif isinstance(interface, definition.Action): - raise Exception('Action plz') + self.interface_type = 'action' + self.subinterfaces = { + 'goal': InterfaceHasher( + interface.goal, includes, + subipath(definition.ACTION_GOAL_SUFFIX)), + 'result': InterfaceHasher( + interface.result, includes, + subipath(definition.ACTION_RESULT_SUFFIX)), + 'feedback': InterfaceHasher( + interface.feedback, includes, + subipath(definition.ACTION_FEEDBACK_SUFFIX)), + 'send_goal_service': InterfaceHasher( + interface.send_goal_service, includes, + subipath(definition.ACTION_GOAL_SERVICE_SUFFIX)), + 'get_result_service': InterfaceHasher( + interface.get_result_service, includes, + subipath(definition.ACTION_RESULT_SERVICE_SUFFIX)), + 'feedback_message': InterfaceHasher( + interface.feedback_message, includes, + subipath(definition.ACTION_FEEDBACK_MESSAGE_SUFFIX)), + } + self.individual_type_description = serialize_individual_action_description(interface) self.json_in = { 'type_description': self.individual_type_description, From 91858742748dd127a01142cad22a4e2eebae3148 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 16:31:12 -0800 Subject: [PATCH 23/65] Simplify subinterface generation Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 154 ++++++------------ 1 file changed, 46 insertions(+), 108 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index e65ff694c..796d3f846 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -59,7 +59,7 @@ def generate_type_hash(generator_arguments_file: str): generate_to_dir = (output_dir / idl_rel_path).parent generate_to_dir.mkdir(parents=True, exist_ok=True) - hasher = InterfaceHasher.from_idl(idl_file, idl_rel_path) + hasher = InterfaceHasher.from_idl(idl_file) generated_files.extend( hasher.write_json_in(output_dir)) hashers.append(hasher) @@ -161,134 +161,72 @@ def serialize_field(member: definition.Member): } -def serialize_individual_type_description(msg: definition.Message): - return { - 'type_name': '/'.join(msg.structure.namespaced_type.namespaced_name()), - 'fields': [serialize_field(member) for member in msg.structure.members] - } - - -def serialize_individual_service_description(srv: definition.Service): - name = srv.namespaced_type.name - members = [ - definition.Member( - definition.NamespacedType( - srv.namespaced_type.namespaces, - name + definition.SERVICE_REQUEST_MESSAGE_SUFFIX), 'request_message'), - definition.Member( - definition.NamespacedType( - srv.namespaced_type.namespaces, - name + definition.SERVICE_RESPONSE_MESSAGE_SUFFIX), 'response_message'), - definition.Member( - definition.NamespacedType( - srv.namespaced_type.namespaces, - name + definition.SERVICE_EVENT_MESSAGE_SUFFIX), 'event_message'), - ] - return { - 'type_name': '/'.join(srv.namespaced_type.namespaced_name()), - 'fields': [serialize_field(member) for member in members], - } - - -def serialize_individual_action_description(action: definition.Action): - name = action.namespaced_type.name - members = [ - definition.Member( - definition.NamespacedType( - action.namespaced_type.namespaces, - name + definition.ACTION_GOAL_SUFFIX), 'goal'), - definition.Member( - definition.NamespacedType( - action.namespaced_type.namespaces, - name + definition.ACTION_RESULT_SUFFIX), 'result'), - definition.Member( - definition.NamespacedType( - action.namespaced_type.namespaces, - name + definition.ACTION_FEEDBACK_SUFFIX), 'feedback'), - definition.Member( - definition.NamespacedType( - action.namespaced_type.namespaces, - name + definition.ACTION_GOAL_SERVICE_SUFFIX), 'send_goal_service'), - definition.Member( - definition.NamespacedType( - action.namespaced_type.namespaces, - name + definition.ACTION_RESULT_SERVICE_SUFFIX), 'get_result_service'), - definition.Member( - definition.NamespacedType( - action.namespaced_type.namespaces, - name + definition.ACTION_FEEDBACK_MESSAGE_SUFFIX), 'feedback_message'), - ] - return { - 'type_name': '/'.join(action.namespaced_type.namespaced_name()), - 'fields': [serialize_field(member) for member in members], - } - - class InterfaceHasher: @classmethod - def from_idl(cls, idl: definition.IdlFile, idl_rel_path: str): + def from_idl(cls, idl: definition.IdlFile): includes = idl.content.get_elements_of_type(definition.Include) for el in idl.content.elements: if isinstance(el, definition.Message): - return InterfaceHasher(el, includes, idl_rel_path) + return InterfaceHasher(el, includes) elif isinstance(el, definition.Service): - return InterfaceHasher(el, includes, idl_rel_path) + return InterfaceHasher(el, includes) elif isinstance(el, definition.Action): - return InterfaceHasher(el, includes, idl_rel_path) + return InterfaceHasher(el, includes) raise Exception('No interface found in IDL') - def __init__(self, interface, includes, idl_rel_path: str): - self.includes = [str(Path(include.locator).with_suffix('.json.in')) for include in includes] + def __init__(self, interface, includes): + self.includes = [ + str(Path(include.locator).with_suffix('.json.in')) + for include in includes] self.interface = interface - self.interface_type = '' - self.rel_path = idl_rel_path self.subinterfaces = {} - def subipath(suffix): - return idl_rel_path.parent / (idl_rel_path.stem + suffix) - if isinstance(interface, definition.Message): + self.namespaced_type = interface.structure.namespaced_type self.interface_type = 'message' - self.individual_type_description = serialize_individual_type_description(interface) + self.individual_type_description = { + 'type_name': '/'.join(self.namespaced_type.namespaced_name()), + 'fields': [ + serialize_field(member) for member in interface.structure.members + ] + } elif isinstance(interface, definition.Service): + self.namespaced_type = interface.namespaced_type self.interface_type = 'service' self.subinterfaces = { - 'request_message': InterfaceHasher( - interface.request_message, includes, - subipath(definition.SERVICE_REQUEST_MESSAGE_SUFFIX)), - 'response_message': InterfaceHasher( - interface.response_message, includes, - subipath(definition.SERVICE_RESPONSE_MESSAGE_SUFFIX)), - 'event_message': InterfaceHasher( - interface.event_message, includes, - subipath(definition.SERVICE_EVENT_MESSAGE_SUFFIX)), + 'request_message': InterfaceHasher(interface.request_message, includes), + 'response_message': InterfaceHasher(interface.response_message, includes), + 'event_message': InterfaceHasher(interface.event_message, includes), + } + self.individual_type_description = { + 'type_name': '/'.join(self.namespaced_type.namespaced_name()), + 'fields': [ + serialize_field(definition.Member(hasher.namespaced_type, field_name)) + for field_name, hasher in self.subinterfaces.items() + ] } - self.individual_type_description = serialize_individual_service_description(interface) elif isinstance(interface, definition.Action): + self.namespaced_type = interface.namespaced_type self.interface_type = 'action' self.subinterfaces = { - 'goal': InterfaceHasher( - interface.goal, includes, - subipath(definition.ACTION_GOAL_SUFFIX)), - 'result': InterfaceHasher( - interface.result, includes, - subipath(definition.ACTION_RESULT_SUFFIX)), - 'feedback': InterfaceHasher( - interface.feedback, includes, - subipath(definition.ACTION_FEEDBACK_SUFFIX)), - 'send_goal_service': InterfaceHasher( - interface.send_goal_service, includes, - subipath(definition.ACTION_GOAL_SERVICE_SUFFIX)), - 'get_result_service': InterfaceHasher( - interface.get_result_service, includes, - subipath(definition.ACTION_RESULT_SERVICE_SUFFIX)), - 'feedback_message': InterfaceHasher( - interface.feedback_message, includes, - subipath(definition.ACTION_FEEDBACK_MESSAGE_SUFFIX)), + 'goal': InterfaceHasher(interface.goal, includes), + 'result': InterfaceHasher(interface.result, includes), + 'feedback': InterfaceHasher(interface.feedback, includes), + 'send_goal_service': InterfaceHasher(interface.send_goal_service, includes), + 'get_result_service': InterfaceHasher(interface.get_result_service, includes), + 'feedback_message': InterfaceHasher(interface.feedback_message, includes), + } + self.individual_type_description = { + 'type_name': '/'.join(self.namespaced_type.namespaced_name()), + 'fields': [ + serialize_field(definition.Member(hasher.namespaced_type, field_name)) + for field_name, hasher in self.subinterfaces.items() + ] } - self.individual_type_description = serialize_individual_action_description(interface) + self.rel_path = Path(*self.namespaced_type.namespaced_name()[1:]) + self.include_path = Path(*self.namespaced_type.namespaced_name()) self.json_in = { 'type_description': self.individual_type_description, 'includes': self.includes, @@ -298,7 +236,7 @@ def write_json_in(self, output_dir): for key, val in self.subinterfaces.items(): val.write_json_in(output_dir) - json_path = (output_dir / self.rel_path).with_suffix('.json.in') + json_path = output_dir / self.rel_path.with_suffix('.json.in') json_path.parent.mkdir(parents=True, exist_ok=True) with json_path.open('w', encoding='utf-8') as json_file: json_file.write(json.dumps(self.json_in, indent=2)) @@ -331,7 +269,7 @@ def write_json_out(self, output_dir, includes_map): loaded_includes.values(), key=lambda td: td['type_name']) } - json_path = (output_dir / self.rel_path).with_suffix('.json') + json_path = output_dir / self.rel_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: json_file.write(json.dumps(self.json_out, indent=2)) return [str(json_path)] @@ -352,7 +290,7 @@ def calculate_hash(self): def write_hash(self, output_dir): type_hash = self.calculate_hash() - hash_path = (output_dir / self.rel_path).with_suffix('.sha256') + hash_path = output_dir / self.rel_path.with_suffix('.sha256') with hash_path.open('w', encoding='utf-8') as hash_file: hash_file.write(json.dumps(type_hash, indent=2)) return [str(hash_path)] From 70c5903cf15fbc6815d37911179983d12495d6a8 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 16:37:27 -0800 Subject: [PATCH 24/65] dedupe repetition in individual type description serializing Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 796d3f846..092bdb5a3 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -161,6 +161,15 @@ def serialize_field(member: definition.Member): } +def serialize_individual_type_description( + namespaced_type: definition.NamespacedType, members: List[definition.Member] +): + return { + 'type_name': '/'.join(namespaced_type.namespaced_name()), + 'fields': [serialize_field(member) for member in members] + } + + class InterfaceHasher: @classmethod @@ -185,12 +194,8 @@ def __init__(self, interface, includes): if isinstance(interface, definition.Message): self.namespaced_type = interface.structure.namespaced_type self.interface_type = 'message' - self.individual_type_description = { - 'type_name': '/'.join(self.namespaced_type.namespaced_name()), - 'fields': [ - serialize_field(member) for member in interface.structure.members - ] - } + self.individual_type_description = serialize_individual_type_description( + self.namespaced_type, interface.structure.members) elif isinstance(interface, definition.Service): self.namespaced_type = interface.namespaced_type self.interface_type = 'service' @@ -199,13 +204,11 @@ def __init__(self, interface, includes): 'response_message': InterfaceHasher(interface.response_message, includes), 'event_message': InterfaceHasher(interface.event_message, includes), } - self.individual_type_description = { - 'type_name': '/'.join(self.namespaced_type.namespaced_name()), - 'fields': [ - serialize_field(definition.Member(hasher.namespaced_type, field_name)) + self.individual_type_description = serialize_individual_type_description( + self.namespaced_type, [ + definition.Member(hasher.namespaced_type, field_name) for field_name, hasher in self.subinterfaces.items() - ] - } + ]) elif isinstance(interface, definition.Action): self.namespaced_type = interface.namespaced_type self.interface_type = 'action' @@ -217,13 +220,11 @@ def __init__(self, interface, includes): 'get_result_service': InterfaceHasher(interface.get_result_service, includes), 'feedback_message': InterfaceHasher(interface.feedback_message, includes), } - self.individual_type_description = { - 'type_name': '/'.join(self.namespaced_type.namespaced_name()), - 'fields': [ - serialize_field(definition.Member(hasher.namespaced_type, field_name)) + self.individual_type_description = serialize_individual_type_description( + self.namespaced_type, [ + definition.Member(hasher.namespaced_type, field_name) for field_name, hasher in self.subinterfaces.items() - ] - } + ]) self.rel_path = Path(*self.namespaced_type.namespaced_name()[1:]) self.include_path = Path(*self.namespaced_type.namespaced_name()) From deade420720963443fba166292d45debd67b80d3 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 17:10:10 -0800 Subject: [PATCH 25/65] Individually filtered includes for all subinterfaces Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 81 +++++++++++-------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 092bdb5a3..29a9c747b 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -16,7 +16,7 @@ import json from pathlib import Path import sys -from typing import List, Tuple +from typing import List from rosidl_parser import definition from rosidl_parser.parser import parse_idl_file @@ -174,78 +174,91 @@ class InterfaceHasher: @classmethod def from_idl(cls, idl: definition.IdlFile): - includes = idl.content.get_elements_of_type(definition.Include) for el in idl.content.elements: if isinstance(el, definition.Message): - return InterfaceHasher(el, includes) + return InterfaceHasher(el) elif isinstance(el, definition.Service): - return InterfaceHasher(el, includes) + return InterfaceHasher(el) elif isinstance(el, definition.Action): - return InterfaceHasher(el, includes) + return InterfaceHasher(el) raise Exception('No interface found in IDL') - def __init__(self, interface, includes): - self.includes = [ - str(Path(include.locator).with_suffix('.json.in')) - for include in includes] + def __init__(self, interface): self.interface = interface self.subinterfaces = {} if isinstance(interface, definition.Message): self.namespaced_type = interface.structure.namespaced_type self.interface_type = 'message' - self.individual_type_description = serialize_individual_type_description( - self.namespaced_type, interface.structure.members) + self.members = interface.structure.members elif isinstance(interface, definition.Service): self.namespaced_type = interface.namespaced_type self.interface_type = 'service' self.subinterfaces = { - 'request_message': InterfaceHasher(interface.request_message, includes), - 'response_message': InterfaceHasher(interface.response_message, includes), - 'event_message': InterfaceHasher(interface.event_message, includes), + 'request_message': InterfaceHasher(interface.request_message), + 'response_message': InterfaceHasher(interface.response_message), + 'event_message': InterfaceHasher(interface.event_message), } - self.individual_type_description = serialize_individual_type_description( - self.namespaced_type, [ - definition.Member(hasher.namespaced_type, field_name) - for field_name, hasher in self.subinterfaces.items() - ]) + self.members = [ + definition.Member(hasher.namespaced_type, field_name) + for field_name, hasher in self.subinterfaces.items() + ] elif isinstance(interface, definition.Action): self.namespaced_type = interface.namespaced_type self.interface_type = 'action' self.subinterfaces = { - 'goal': InterfaceHasher(interface.goal, includes), - 'result': InterfaceHasher(interface.result, includes), - 'feedback': InterfaceHasher(interface.feedback, includes), - 'send_goal_service': InterfaceHasher(interface.send_goal_service, includes), - 'get_result_service': InterfaceHasher(interface.get_result_service, includes), - 'feedback_message': InterfaceHasher(interface.feedback_message, includes), + 'goal': InterfaceHasher(interface.goal), + 'result': InterfaceHasher(interface.result), + 'feedback': InterfaceHasher(interface.feedback), + 'send_goal_service': InterfaceHasher(interface.send_goal_service), + 'get_result_service': InterfaceHasher(interface.get_result_service), + 'feedback_message': InterfaceHasher(interface.feedback_message), } - self.individual_type_description = serialize_individual_type_description( - self.namespaced_type, [ - definition.Member(hasher.namespaced_type, field_name) - for field_name, hasher in self.subinterfaces.items() - ]) + self.members = [ + definition.Member(hasher.namespaced_type, field_name) + for field_name, hasher in self.subinterfaces.items() + ] + + self.individual_type_description = serialize_individual_type_description( + self.namespaced_type, self.members) + included_types = [] + for member in self.members: + if isinstance(member.type, definition.NamespacedType): + included_types.append(member.type) + elif ( + isinstance(member.type, definition.AbstractNestedType) and + isinstance(member.type.value_type, definition.NamespacedType) + ): + included_types.append(member.type.value_type) + + self.includes = [ + str(Path(*t.namespaced_name()).with_suffix('.json.in')) + for t in included_types + ] self.rel_path = Path(*self.namespaced_type.namespaced_name()[1:]) self.include_path = Path(*self.namespaced_type.namespaced_name()) + self.json_in = { 'type_description': self.individual_type_description, 'includes': self.includes, } def write_json_in(self, output_dir): + generated_files = [] for key, val in self.subinterfaces.items(): - val.write_json_in(output_dir) + generated_files += val.write_json_in(output_dir) json_path = output_dir / self.rel_path.with_suffix('.json.in') json_path.parent.mkdir(parents=True, exist_ok=True) with json_path.open('w', encoding='utf-8') as json_file: json_file.write(json.dumps(self.json_in, indent=2)) - return [str(json_path)] + return generated_files + [str(json_path)] def write_json_out(self, output_dir, includes_map): + generated_files = [] for key, val in self.subinterfaces.items(): - val.write_json_out(output_dir, includes_map) + generated_files += val.write_json_out(output_dir, includes_map) pending_includes = self.json_in['includes'] loaded_includes = {} @@ -273,7 +286,7 @@ def write_json_out(self, output_dir, includes_map): json_path = output_dir / self.rel_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: json_file.write(json.dumps(self.json_out, indent=2)) - return [str(json_path)] + return generated_files + [str(json_path)] def calculate_hash(self): json_out_repr = json.dumps(self.json_out) From b7600fe242a9d81f3338abdb8f3d12ce5a452bc3 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 17:10:30 -0800 Subject: [PATCH 26/65] No indent on json outs Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 29a9c747b..9b5a2ebc6 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -285,7 +285,7 @@ def write_json_out(self, output_dir, includes_map): json_path = output_dir / self.rel_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json.dumps(self.json_out, indent=2)) + json_file.write(json.dumps(self.json_out)) return generated_files + [str(json_path)] def calculate_hash(self): From b3fb44419f9c8c55ca04b4dacc10567e8c1b0e19 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 19:31:29 -0800 Subject: [PATCH 27/65] Ad dtoplevel for action and service Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 2 ++ rosidl_generator_cpp/resource/action__struct.hpp.em | 2 ++ rosidl_pycommon/rosidl_pycommon/__init__.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index fc289a729..d797475d2 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -91,6 +91,8 @@ TEMPLATE( from rosidl_parser.definition import Action }@ @[for action in content.get_elements_of_type(Action)]@ +static const @(TYPE_HASH(idl_structure_type_to_c_typename(action.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['action']))@ + @{ TEMPLATE( 'msg__struct.h.em', diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index f6527f85f..3c7d4fdb6 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -78,6 +78,8 @@ namespace @(ns) @[end for]@ struct @(action.namespaced_type.name) { + static constexpr @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['action'], indent=2))@ + /// The goal message defined in the action definition. using Goal = @(action_name)@(ACTION_GOAL_SUFFIX); /// The result message defined in the action definition. diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index ff4abaf37..12960604a 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -227,6 +227,6 @@ def _expand_type_hash(variable_name, hex_string, indent=0): for i in range(32): if i % 8 == 0: result += f'\n{ind_str} ' - result += f'0x{hex_string[i*2:i*2+1]}, ' + result += f'0x{hex_string[i*2:i*2+2]}, ' result += f'\n{ind_str}}};\n' return result From 27fce61c1d8b8dd91051f2a2abe12f5b2ebc9519 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Feb 2023 20:13:43 -0800 Subject: [PATCH 28/65] Code cleanup pass Signed-off-by: Emerson Knapp --- rosidl_generator_type_hash/CMakeLists.txt | 2 +- ...erator_type_hash_generate_interfaces.cmake | 1 - .../rosidl_generator_type_hash/__init__.py | 34 +++++++++++-------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/rosidl_generator_type_hash/CMakeLists.txt b/rosidl_generator_type_hash/CMakeLists.txt index 432d3cddd..45061e95e 100644 --- a/rosidl_generator_type_hash/CMakeLists.txt +++ b/rosidl_generator_type_hash/CMakeLists.txt @@ -22,6 +22,6 @@ install( DESTINATION lib/${PROJECT_NAME} ) install( - DIRECTORY cmake # resource + DIRECTORY cmake DESTINATION share/${PROJECT_NAME} ) diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index c41222c2c..c8d1a6d59 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -34,7 +34,6 @@ endforeach() set(_dependency_files "") set(_dependency_paths "") -set(_dependencies "") foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) set(_include_path "${${_pkg_name}_DIR}/..") normalize_path(_include_path "${_include_path}") diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 9b5a2ebc6..25f8d46f4 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -22,7 +22,7 @@ from rosidl_parser.parser import parse_idl_file -def generate_type_hash(generator_arguments_file: str): +def generate_type_hash(generator_arguments_file: str) -> List[str]: with open(generator_arguments_file, 'r') as f: args = json.load(f) package_name = args['package_name'] @@ -113,7 +113,7 @@ def generate_type_hash(generator_arguments_file: str): } -def serialize_field_type(ftype: definition.AbstractType): +def serialize_field_type(ftype: definition.AbstractType) -> dict: result = { 'type_id': 0, 'length': 0, @@ -121,26 +121,26 @@ def serialize_field_type(ftype: definition.AbstractType): 'nested_type_name': '', } + # Determine value type, if this is a nested type type_id_offset = 0 if isinstance(ftype, definition.AbstractNestableType): value_type = ftype elif isinstance(ftype, definition.AbstractNestedType): type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] value_type = ftype.value_type - if ftype.has_maximum_size(): - try: - result['length'] = ftype.maximum_size - except AttributeError: - result['length'] = ftype.size else: raise Exception('Unable to translate field type', ftype) + # Translate value type to FieldType.msg const value if isinstance(value_type, definition.BasicType): result['type_id'] = FIELD_TYPES[value_type.typename] + type_id_offset elif isinstance(value_type, definition.AbstractGenericString): result['type_id'] = FIELD_TYPES[type(value_type)] + type_id_offset if value_type.has_maximum_size(): - result['string_length'] = value_type.maximum_size + try: + result['length'] = value_type.maximum_size + except AttributeError: + result['length'] = value_type.size elif isinstance(value_type, definition.NamespacedType): result['type_id'] = type_id_offset result['nested_type_name'] = '/'.join(value_type.namespaced_name()) @@ -153,7 +153,7 @@ def serialize_field_type(ftype: definition.AbstractType): return result -def serialize_field(member: definition.Member): +def serialize_field(member: definition.Member) -> dict: return { 'name': member.name, 'type': serialize_field_type(member.type), @@ -163,7 +163,7 @@ def serialize_field(member: definition.Member): def serialize_individual_type_description( namespaced_type: definition.NamespacedType, members: List[definition.Member] -): +) -> dict: return { 'type_name': '/'.join(namespaced_type.namespaced_name()), 'fields': [serialize_field(member) for member in members] @@ -187,6 +187,7 @@ def __init__(self, interface): self.interface = interface self.subinterfaces = {} + # Determine top level interface, and member fields based on that if isinstance(interface, definition.Message): self.namespaced_type = interface.structure.namespaced_type self.interface_type = 'message' @@ -221,6 +222,8 @@ def __init__(self, interface): self.individual_type_description = serialize_individual_type_description( self.namespaced_type, self.members) + + # Determine needed includes from member fields included_types = [] for member in self.members: if isinstance(member.type, definition.NamespacedType): @@ -244,7 +247,7 @@ def __init__(self, interface): 'includes': self.includes, } - def write_json_in(self, output_dir): + def write_json_in(self, output_dir) -> List[str]: generated_files = [] for key, val in self.subinterfaces.items(): generated_files += val.write_json_in(output_dir) @@ -255,12 +258,13 @@ def write_json_in(self, output_dir): json_file.write(json.dumps(self.json_in, indent=2)) return generated_files + [str(json_path)] - def write_json_out(self, output_dir, includes_map): + def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: generated_files = [] for key, val in self.subinterfaces.items(): generated_files += val.write_json_out(output_dir, includes_map) - pending_includes = self.json_in['includes'] + # Recursively load includes from all included type descriptions + pending_includes = self.includes[:] loaded_includes = {} while pending_includes: process_include = pending_includes.pop() @@ -288,7 +292,7 @@ def write_json_out(self, output_dir, includes_map): json_file.write(json.dumps(self.json_out)) return generated_files + [str(json_path)] - def calculate_hash(self): + def calculate_hash(self) -> dict: json_out_repr = json.dumps(self.json_out) sha = hashlib.sha256() sha.update(json_out_repr.encode('utf-8')) @@ -302,7 +306,7 @@ def calculate_hash(self): return type_hash_infos - def write_hash(self, output_dir): + def write_hash(self, output_dir: Path) -> List[str]: type_hash = self.calculate_hash() hash_path = output_dir / self.rel_path.with_suffix('.sha256') with hash_path.open('w', encoding='utf-8') as hash_file: From 8fc293a1cc9e6006e286bd55f9bf4367c61d3084 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 16 Feb 2023 21:37:50 -0800 Subject: [PATCH 29/65] Need extra declaration for constexpr static template member Signed-off-by: Emerson Knapp --- rosidl_generator_cpp/resource/action__struct.hpp.em | 2 +- rosidl_generator_cpp/resource/msg__struct.hpp.em | 6 +++++- rosidl_generator_cpp/resource/srv__struct.hpp.em | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index 3c7d4fdb6..f34c54014 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -78,7 +78,7 @@ namespace @(ns) @[end for]@ struct @(action.namespaced_type.name) { - static constexpr @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['action'], indent=2))@ + static constexpr const @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['action'], indent=2))@ /// The goal message defined in the action definition. using Goal = @(action_name)@(ACTION_GOAL_SUFFIX); diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index 0f54930fb..c2bfbda56 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -96,13 +96,14 @@ namespace @(ns) @[end for]@ // message struct + template struct @(message.structure.namespaced_type.name)_ { using Type = @(message.structure.namespaced_type.name)_; // Type Version Hash for interface - static constexpr @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['message'], indent=2))@ + constexpr static const @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['message'], indent=2))@ @{ # The creation of the constructors for messages is a bit complicated. The goal @@ -360,6 +361,9 @@ u@ using @(message.structure.namespaced_type.name) = @(message_typename)_>; +template +constexpr const uint8_t @(message.structure.namespaced_type.name)_::TYPE_VERSION_HASH[32]; + // constant definitions @[for c in message.constants]@ @[ if c.name in msvc_common_macros]@ diff --git a/rosidl_generator_cpp/resource/srv__struct.hpp.em b/rosidl_generator_cpp/resource/srv__struct.hpp.em index 3b0cd166c..93a8222bb 100644 --- a/rosidl_generator_cpp/resource/srv__struct.hpp.em +++ b/rosidl_generator_cpp/resource/srv__struct.hpp.em @@ -34,7 +34,7 @@ struct @(service.namespaced_type.name) @{ service_typename = '::'.join(service.namespaced_type.namespaced_name()) }@ - static constexpr @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['service'], indent=2))@ + static constexpr const @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['service'], indent=2))@ using Request = @(service_typename)_Request; using Response = @(service_typename)_Response; From 9977f340fe5b073ba327397d3feb2ab6e243d1d0 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 20 Feb 2023 12:08:34 -0800 Subject: [PATCH 30/65] Add type hash to introspection Signed-off-by: Emerson Knapp --- .../rosidl_typesupport_introspection_c/message_introspection.h | 2 ++ .../resource/msg__type_support.c.em | 1 + .../message_introspection.hpp | 1 + .../resource/msg__type_support.cpp.em | 1 + 4 files changed, 5 insertions(+) diff --git a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h index d5d220e08..e631c03a9 100644 --- a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h +++ b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h @@ -85,6 +85,8 @@ typedef struct rosidl_typesupport_introspection_c__MessageMembers_s const char * message_namespace_; /// The name of the interface, e.g. "Int16" const char * message_name_; + /// + const uint8_t * type_hash_; /// The number of fields in the interface uint32_t member_count_; /// The size of the interface structure in memory diff --git a/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em b/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em index f0004338a..58c48b813 100644 --- a/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em +++ b/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em @@ -270,6 +270,7 @@ for index, member in enumerate(message.structure.members): static const rosidl_typesupport_introspection_c__MessageMembers @(function_prefix)__@(message.structure.namespaced_type.name)_message_members = { "@('__'.join([package_name] + list(interface_path.parents[0].parts)))", // message namespace "@(message.structure.namespaced_type.name)", // message name + @('__'.join(message.structure.namespaced_type.namespaced_name()))__TYPE_VERSION_HASH, @(len(message.structure.members)), // number of fields sizeof(@('__'.join([package_name] + list(interface_path.parents[0].parts) + [message.structure.namespaced_type.name]))), @(function_prefix)__@(message.structure.namespaced_type.name)_message_member_array, // message members diff --git a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp index 2340f2c8c..8103217a3 100644 --- a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp +++ b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp @@ -92,6 +92,7 @@ typedef struct ROSIDL_TYPESUPPORT_INTROSPECTION_CPP_PUBLIC MessageMembers_s const char * message_namespace_; /// The name of the interface, e.g. "Int16" const char * message_name_; + const uint8_t * type_hash_; /// The number of fields in the interface uint32_t member_count_; /// The size of the interface structure in memory diff --git a/rosidl_typesupport_introspection_cpp/resource/msg__type_support.cpp.em b/rosidl_typesupport_introspection_cpp/resource/msg__type_support.cpp.em index 2ebcc9274..7c1cc636b 100644 --- a/rosidl_typesupport_introspection_cpp/resource/msg__type_support.cpp.em +++ b/rosidl_typesupport_introspection_cpp/resource/msg__type_support.cpp.em @@ -235,6 +235,7 @@ for index, member in enumerate(message.structure.members): static const ::rosidl_typesupport_introspection_cpp::MessageMembers @(message.structure.namespaced_type.name)_message_members = { "@('::'.join([package_name] + list(interface_path.parents[0].parts)))", // message namespace "@(message.structure.namespaced_type.name)", // message name + @('::'.join(message.structure.namespaced_type.namespaced_name()))::TYPE_VERSION_HASH, @(len(message.structure.members)), // number of fields sizeof(@('::'.join([package_name] + list(interface_path.parents[0].parts) + [message.structure.namespaced_type.name]))), @(message.structure.namespaced_type.name)_message_member_array, // message members From 0d2a67f7942e1dc78f2aca6ac7c5366107383d37 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 20 Feb 2023 16:25:51 -0800 Subject: [PATCH 31/65] Service introspection type hash todo Signed-off-by: Emerson Knapp --- .../rosidl_typesupport_introspection_c/service_introspection.h | 2 ++ .../service_introspection.hpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h index ea57dffd6..007569800 100644 --- a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h +++ b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h @@ -32,6 +32,8 @@ typedef struct rosidl_typesupport_introspection_c__ServiceMembers_s const char * service_namespace_; /// The name of the service, e.g. "AddTwoInts" const char * service_name_; + /// TODO(emersonknapp) + // const uint8_t * type_hash_; /// A pointer to the introspection information structure for the request interface. const rosidl_typesupport_introspection_c__MessageMembers * request_members_; /// A pointer to the introspection information structure for the response interface. diff --git a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp index cad7e04e8..187e29fd5 100644 --- a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp +++ b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp @@ -35,6 +35,8 @@ typedef struct ServiceMembers_s const char * service_namespace_; /// The name of the service, e.g. "AddTwoInts" const char * service_name_; + /// TODO(emersonknapp) + // const uint8_t * service_type_hash_; /// A pointer to the introspection information structure for the request interface. const MessageMembers * request_members_; /// A pointer to the introspection information structure for the response interface. From f6a50b769e6ee5e5415f2fabbf3382302b339255 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 20 Feb 2023 23:37:49 -0800 Subject: [PATCH 32/65] Remove CLI Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash_generate_interfaces.cmake | 4 ++++ rosidl_generator_type_hash/setup.cfg | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 rosidl_generator_type_hash/setup.cfg diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index c8d1a6d59..5f3ea0277 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -19,6 +19,8 @@ set(_generated_json_in "") set(_generated_json "") set(_generated_hash_tuples "") set(_generated_hash_files "") + +# Create lists of generated fiiles foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) get_filename_component(_parent_folder "${_abs_idl_file}" DIRECTORY) get_filename_component(_parent_folder "${_parent_folder}" NAME) @@ -32,6 +34,7 @@ foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) list(APPEND _generated_hash_tuples "${_parent_folder}/${_idl_stem}:${_hash_file}") endforeach() +# Find dependency packages' .json.in files set(_dependency_files "") set(_dependency_paths "") foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) @@ -44,6 +47,7 @@ foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) endforeach() endforeach() +# Export __HASH_TUPLES variable for use by dependents set(${rosidl_generate_interfaces_TARGET}__HASH_TUPLES ${_generated_hash_tuples}) set(target_dependencies diff --git a/rosidl_generator_type_hash/setup.cfg b/rosidl_generator_type_hash/setup.cfg deleted file mode 100644 index b62f1af69..000000000 --- a/rosidl_generator_type_hash/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[options.entry_points] -rosidl_cli.command.generate.type_extensions = - type_hash = rosidl_generator_type_hash.cli:GenerateTypeHash From ec75b02796834c1f74fd532d0b9311772cee124e Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 13:52:05 -0800 Subject: [PATCH 33/65] Jsonschemas first pass Signed-off-by: Emerson Knapp --- README.md | 2 + rosidl_generator_type_hash/README.md | 37 ++++++++++++ ...erator_type_hash_generate_interfaces.cmake | 6 +- .../IndividualTypeDescription.schema.json | 37 ++++++++++++ .../resource/TypeDescription.schema.json | 15 +++++ .../resource/TypeDescriptionIn.schema.json | 15 +++++ .../resource/TypeVersionHash.schema.json | 57 +++++++++++++++++++ .../rosidl_generator_type_hash/__init__.py | 12 ++-- 8 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 rosidl_generator_type_hash/README.md create mode 100644 rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json create mode 100644 rosidl_generator_type_hash/resource/TypeDescription.schema.json create mode 100644 rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json create mode 100644 rosidl_generator_type_hash/resource/TypeVersionHash.schema.json diff --git a/README.md b/README.md index e2d1ef3cb..b556c1249 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ See [documentation](https://docs.ros.org/en/rolling/Concepts/About-Internal-Inte * Generate the ROS interfaces in C * [rosidl_generator_cpp](./rosidl_generator_cpp) * Generate the ROS interfaces in C++ +* [rosidl_generator_type_hash](./rosidl_generator_type_hash) + * Generate SHA256 hash values for ROS 2 interface descriptions for use by other generators * [rosidl_parser](./rosidl_parser) * Parser for `.idl` ROS interface files * [rosidl_runtime_c](./rosidl_runtime_c) diff --git a/rosidl_generator_type_hash/README.md b/rosidl_generator_type_hash/README.md new file mode 100644 index 000000000..18cf4d73f --- /dev/null +++ b/rosidl_generator_type_hash/README.md @@ -0,0 +1,37 @@ +# rosidl_generator_type_hash + +This generator serializes ROS 2 interface descriptions (message, service, action) to a common format and uses SHA256 to hash that representation into a unique hash for each type. + +The SHA256 hashes generated by this package must match those generated by `rcl_calculate_type_version_hash`. The `.json` files generated must, therefore, match the result of `rcl_type_description_to_hashable_json`. + +## Generated files + +This generator creates the following output files from parsed IDL interface descriptions: + +* `interface_type.sha256.json` +* `interface_type.in.json` +* `interface_type.json` + +See the schema files for an exact description for the contents of the generated files. + +### `.sha256.json` + +**Schema**: [TypeVersionHash](./resource/TypeVersionHash.schema.json) + +The true output of the generator, containing hashes for top-level interface and sub-interfaces if any are present, to be consumed by code generators. + +The remaining generated files are intermediate artifacts that lead to the output hashes. + +### `.json` + +**Schema**: [TypeDescription](./resource/TypeDescription.schema.json) + +The fully-expanded serialized equivalent of `type_description_interfaces/msg/TypeDescription`, whose contents are hashed. + +NOTE: `.json` contents must have all whitespace stripped before hashing. This package generates json without whitespace, including having no trailing newline. + +### `.in.json` + +**Schema**: [TypeDescriptionIn](./resource/TypeDescriptionIn.schema.json) + +The `IndividualTypeDescription`-equivalent serialization, and a list of referenced `.in.json` files to be used when expanding to `.json`. This file is used by dependent interfaces, especially in dependent packages, when expanding to `.json` so IDL files must only be parsed a single time when running the generator on their own package. diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index 5f3ea0277..6c4185dc4 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -26,15 +26,15 @@ foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) get_filename_component(_parent_folder "${_parent_folder}" NAME) get_filename_component(_idl_stem "${_abs_idl_file}" NAME_WE) list(APPEND _generated_json_in - "${_output_path}/${_parent_folder}/${_idl_stem}.json.in") + "${_output_path}/${_parent_folder}/${_idl_stem}.in.json") list(APPEND _generated_json "${_output_path}/${_parent_folder}/${_idl_stem}.json") - set(_hash_file "${_output_path}/${_parent_folder}/${_idl_stem}.sha256") + set(_hash_file "${_output_path}/${_parent_folder}/${_idl_stem}.sha256.json") list(APPEND _generated_hash_files ${_hash_file}) list(APPEND _generated_hash_tuples "${_parent_folder}/${_idl_stem}:${_hash_file}") endforeach() -# Find dependency packages' .json.in files +# Find dependency packages' .in.json files set(_dependency_files "") set(_dependency_paths "") foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) diff --git a/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json b/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json new file mode 100644 index 000000000..24149d251 --- /dev/null +++ b/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "IndividualTypeDescription.jsonschema", + "title": "TypeVersionHash", + "description": "IndividualTypeDescription", + "type": "object", + "properties": { + "type_name": {"type": "string", "maxLength": 255}, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + } + } + }, + "required": ["type_name", "fields"], + "$defs": { + "Field": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "type": {"$ref": "#/$defs/FieldType"} + }, + "required": ["name", "type"] + }, + "FieldType": { + "type": "object", + "properties": { + "type_id": {"type": "integer", "minimum": 0, "maximum": 255}, + "length": {"type": "integer", "minimum": 0}, + "string_length": {"type": "integer", "minimum": 0}, + "nested_type_name": {"type": "string", "maxLength": 255} + }, + "required": ["type_id", "length", "string_length", "nested_type_name"] + } + } +} diff --git a/rosidl_generator_type_hash/resource/TypeDescription.schema.json b/rosidl_generator_type_hash/resource/TypeDescription.schema.json new file mode 100644 index 000000000..2918c08ec --- /dev/null +++ b/rosidl_generator_type_hash/resource/TypeDescription.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "TypeDescription.schema.json", + "title": "TypeVersionHash", + "description": "IndividualTypeDescription", + "type": "object", + "properties": { + "type_description": {"$ref": "IndividualTypeDescription"}, + "referenced_type_descriptions": { + "type": "array", + "items": { "$ref": "IndividualTypeDescription" } + } + }, + "required": ["type_description", "referenced_type_descriptions"] +} diff --git a/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json b/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json new file mode 100644 index 000000000..9c0bd2034 --- /dev/null +++ b/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "TypeDescription.schema.json", + "title": "TypeVersionHash", + "description": "IndividualTypeDescription", + "type": "object", + "properties": { + "type_description": {"$ref": "IndividualTypeDescription"}, + "referenced_type_descriptions": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["type_description", "referenced_type_descriptions"] +} diff --git a/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json b/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json new file mode 100644 index 000000000..4fa19c8ba --- /dev/null +++ b/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "TypeVersionHash.jsonschema", + "title": "TypeVersionHash", + "description": "A tree of hashes (and sub-hashes) for ROS 2 interfaces", + "type": "object", + "$defs": { + "message": { + "type": "object", + "properties": { + "message": { "type": "string" } + }, + "required": [ "message" ] + }, + "service": { + "type": "object", + "properties": { + "service": { "type": "string" }, + "request_message": { "$ref": "#/$defs/message" }, + "response_message": { "$ref": "#/$defs/message" }, + "event_message": { "$ref": "#/$defs/message" } + }, + "required": [ + "service", + "request_message", + "response_message", + "event_message" + ] + }, + "action": { + "type": "object", + "properties": { + "action": { "type": "string" }, + "goal": { "$ref": "#/$defs/message" }, + "result": { "$ref": "#/$defs/message" }, + "feedback": { "$ref": "#/$defs/message" }, + "send_goal_service": { "$ref": "#/$defs/service" }, + "get_result_service": { "$ref": "#/$defs/service" }, + "feedback_message": { "$ref": "#/$defs/message" } + }, + "required": [ + "action", + "goal", + "result", + "feedback", + "send_goal_service", + "get_result_service", + "feedback_message" + ] + } + }, + "oneOf": [ + { "$ref": "#/$defs/message" }, + { "$ref": "#/$defs/service" }, + { "$ref": "#/$defs/action" } + ] +} diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 25f8d46f4..1367f3b91 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -30,7 +30,7 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: idl_tuples = args['idl_tuples'] include_paths = args.get('include_paths', []) - # Lookup for directory containing pregenerated .json.in files + # Lookup for directory containing pregenerated .in.json files include_map = { package_name: output_dir } @@ -43,7 +43,7 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: generated_files = [] hashers = [] - # First generate all .json.in files (so referenced types can be used in expansion) + # First generate all .in.json files (so referenced types can be used in expansion) for idl_tuple in idl_tuples: idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 @@ -64,7 +64,7 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: hasher.write_json_in(output_dir)) hashers.append(hasher) - # Expand .json.in and generate .sha256 hash files + # Expand .in.json and generate .sha256.json hash files for hasher in hashers: generated_files.extend( hasher.write_json_out(output_dir, include_map)) @@ -235,7 +235,7 @@ def __init__(self, interface): included_types.append(member.type.value_type) self.includes = [ - str(Path(*t.namespaced_name()).with_suffix('.json.in')) + str(Path(*t.namespaced_name()).with_suffix('.in.json')) for t in included_types ] @@ -252,7 +252,7 @@ def write_json_in(self, output_dir) -> List[str]: for key, val in self.subinterfaces.items(): generated_files += val.write_json_in(output_dir) - json_path = output_dir / self.rel_path.with_suffix('.json.in') + json_path = output_dir / self.rel_path.with_suffix('.in.json') json_path.parent.mkdir(parents=True, exist_ok=True) with json_path.open('w', encoding='utf-8') as json_file: json_file.write(json.dumps(self.json_in, indent=2)) @@ -308,7 +308,7 @@ def calculate_hash(self) -> dict: def write_hash(self, output_dir: Path) -> List[str]: type_hash = self.calculate_hash() - hash_path = output_dir / self.rel_path.with_suffix('.sha256') + hash_path = output_dir / self.rel_path.with_suffix('.sha256.json') with hash_path.open('w', encoding='utf-8') as hash_file: hash_file.write(json.dumps(type_hash, indent=2)) return [str(hash_path)] From ff5135a1579bfd8884fd42cfd128c838e432d871 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 16:37:50 -0800 Subject: [PATCH 34/65] Add rosidl_type_hash_t to rosidl_runtime_c Signed-off-by: Emerson Knapp --- rosidl_runtime_c/CMakeLists.txt | 1 + .../include/rosidl_runtime_c/type_hash.h | 36 +++++++++++++++++++ rosidl_runtime_c/src/type_hash.c | 22 ++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h create mode 100644 rosidl_runtime_c/src/type_hash.c diff --git a/rosidl_runtime_c/CMakeLists.txt b/rosidl_runtime_c/CMakeLists.txt index 33baf9b25..e6b162e8e 100644 --- a/rosidl_runtime_c/CMakeLists.txt +++ b/rosidl_runtime_c/CMakeLists.txt @@ -21,6 +21,7 @@ add_library(${PROJECT_NAME} "src/sequence_bound.c" "src/service_type_support.c" "src/string_functions.c" + "src/type_hash.c" "src/u16string_functions.c" ) target_include_directories(${PROJECT_NAME} PUBLIC diff --git a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h new file mode 100644 index 000000000..643638ee4 --- /dev/null +++ b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h @@ -0,0 +1,36 @@ +// Copyright 2023 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSIDL_RUNTIME_C__TYPE_HASH_H_ +#define ROSIDL_RUNTIME_C__TYPE_HASH_H_ + +#include + +#include "rcutils/sha256.h" + +#include "rosidl_runtime_c/visibility_control.h" + +#define ROSIDL_TYPE_HASH_VERSION_UNSET 0 +#define ROSIDL_TYPE_HASH_SIZE RCUTILS_SHA256_BLOCK_SIZE + +typedef struct rosidl_type_hash_s +{ + uint8_t version; + uint8_t value[ROSIDL_TYPE_HASH_SIZE]; +} rosidl_type_hash_t; + +ROSIDL_GENERATOR_C_PUBLIC +rosidl_type_hash_t get_zero_initialized_type_hash(void); + +#endif // ROSIDL_RUNTIME_C__TYPE_HASH_H_ diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c new file mode 100644 index 000000000..ba256e42b --- /dev/null +++ b/rosidl_runtime_c/src/type_hash.c @@ -0,0 +1,22 @@ +// Copyright 2023 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rosidl_runtime_c/type_hash.h" + +rosidl_type_hash_t +rosidl_get_zero_initialized_type_hash(void) +{ + rosidl_type_hash_t zero_initialized_type_hash = {0}; + return zero_initialized_type_hash; +} From 4d2799871f3a56f5f6d31b6e01815ca36ab25704 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 20:17:38 -0800 Subject: [PATCH 35/65] Type hash struct Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/msg__struct.h.em | 6 ++- rosidl_pycommon/rosidl_pycommon/__init__.py | 12 ++++-- .../include/rosidl_runtime_c/type_hash.h | 16 +++++++- rosidl_runtime_c/src/type_hash.c | 39 +++++++++++++++++++ .../message_introspection.h | 3 +- .../message_introspection.hpp | 3 +- 6 files changed, 70 insertions(+), 9 deletions(-) diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index 5d41ca389..b242c6608 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -28,6 +28,7 @@ from rosidl_generator_c import value_to_c @{ from collections import OrderedDict includes = OrderedDict() +includes.setdefault('rosidl_runtime_c/type_hash.h', []) for member in message.structure.members: if isinstance(member.type, AbstractSequence) and isinstance(member.type.value_type, BasicType): member_names = includes.setdefault( @@ -61,8 +62,6 @@ for member in message.structure.members: }@ @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -// Type Version Hash for interface -static const @(TYPE_HASH(idl_structure_type_to_c_typename(message.structure.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['message']))@ @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< // Constants defined in the message @@ -180,6 +179,9 @@ typedef struct @(idl_structure_type_to_c_typename(message.structure.namespaced_t @(idl_declaration_to_c(member.type, member.name)); @[end for]@ } @(idl_structure_type_to_c_typename(message.structure.namespaced_type)); + +// Type Version Hash for interface +static const @(TYPE_HASH(idl_structure_type_to_c_typename(message.structure.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['message']))@ @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 12960604a..24a8d5733 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -221,12 +221,16 @@ def _expand_template(template_name, **kwargs): interpreter.invoke('afterInclude') -def _expand_type_hash(variable_name, hex_string, indent=0): +def _expand_type_hash(variable_name, hash_string, indent=0): + """ + Generate a rosidl_type_hash_t instance with 8 bytes per line for readability. + """ ind_str = ' ' * indent - result = f'uint8_t {variable_name}[32] = {{' + RIHS_VERSION = 1 # TODO(emersonknapp) where should I get this from.... IS IT IN THE HEX_STRING? + result = f'rosidl_type_hash_t {variable_name} = {{{RIHS_VERSION}, {{' for i in range(32): if i % 8 == 0: result += f'\n{ind_str} ' - result += f'0x{hex_string[i*2:i*2+2]}, ' - result += f'\n{ind_str}}};\n' + result += f'0x{hash_string[i*2:i*2+2]}, ' + result += f'\n{ind_str}}}}};\n' return result diff --git a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h index 643638ee4..17b414a27 100644 --- a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h +++ b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h @@ -17,6 +17,7 @@ #include +#include "rcutils/allocator.h" #include "rcutils/sha256.h" #include "rosidl_runtime_c/visibility_control.h" @@ -31,6 +32,19 @@ typedef struct rosidl_type_hash_s } rosidl_type_hash_t; ROSIDL_GENERATOR_C_PUBLIC -rosidl_type_hash_t get_zero_initialized_type_hash(void); +rosidl_type_hash_t +rosidl_get_zero_initialized_type_hash(void); + +ROSIDL_GENERATOR_C_PUBLIC +rcutils_ret_t +rosidl_stringify_type_hash( + const rosidl_type_hash_t * type_hash, + rcutils_allocator_t allocator, + char ** output_string); + +ROSIDL_GENERATOR_C_PUBLIC +rosidl_type_hash_t +rosidl_parse_type_hash_string(const char * type_hash_string, size_t length); + #endif // ROSIDL_RUNTIME_C__TYPE_HASH_H_ diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index ba256e42b..2edffa0d3 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -14,9 +14,48 @@ #include "rosidl_runtime_c/type_hash.h" +#include "rcutils/error_handling.h" +#include "rcutils/format_string.h" + rosidl_type_hash_t rosidl_get_zero_initialized_type_hash(void) { rosidl_type_hash_t zero_initialized_type_hash = {0}; return zero_initialized_type_hash; } + +rcutils_ret_t +rosidl_stringify_type_hash( + const rosidl_type_hash_t * type_hash, + rcutils_allocator_t allocator, + char ** output_string) +{ + if (!type_hash) { + RCUTILS_SET_ERROR_MSG("Null type_hash argument."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + if (!rcutils_allocator_is_valid(&allocator)) { + RCUTILS_SET_ERROR_MSG("Invalid allocator"); + return RCUTILS_RET_INVALID_ARGUMENT; + } + if (!output_string) { + RCUTILS_SET_ERROR_MSG("Null string destination pointer."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + + // Hash representation will be simple hex string, two characters per byte + const char * fmt = "RIHS%d_%064d"; + char * local_output = rcutils_format_string(allocator, fmt, type_hash->version, 0); + if (!local_output) { + *output_string = NULL; + RCUTILS_SET_ERROR_MSG("Unable to allocate space for stringified type hash."); + return RCUTILS_RET_BAD_ALLOC; + } + for (size_t i = 0; i < RCUTILS_SHA256_BLOCK_SIZE; i++) { + sprintf(local_output + (i * 2), "%02x", type_hash->value[i]); + } + fprintf(stderr, "%s", local_output); + + *output_string = local_output; + return RCUTILS_RET_OK; +} diff --git a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h index e631c03a9..e105a8e1b 100644 --- a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h +++ b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h @@ -21,6 +21,7 @@ #include "rosidl_runtime_c/message_initialization.h" #include "rosidl_runtime_c/message_type_support_struct.h" +#include "rosidl_runtime_c/type_hash.h" #include "rosidl_typesupport_introspection_c/visibility_control.h" @@ -86,7 +87,7 @@ typedef struct rosidl_typesupport_introspection_c__MessageMembers_s /// The name of the interface, e.g. "Int16" const char * message_name_; /// - const uint8_t * type_hash_; + const rosidl_type_hash_t type_hash_; /// The number of fields in the interface uint32_t member_count_; /// The size of the interface structure in memory diff --git a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp index 8103217a3..1407d10fc 100644 --- a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp +++ b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp @@ -19,6 +19,7 @@ #include #include "rosidl_runtime_c/message_type_support_struct.h" +#include "rosidl_runtime_c/type_hash.h" #include "rosidl_runtime_cpp/message_initialization.hpp" @@ -92,7 +93,7 @@ typedef struct ROSIDL_TYPESUPPORT_INTROSPECTION_CPP_PUBLIC MessageMembers_s const char * message_namespace_; /// The name of the interface, e.g. "Int16" const char * message_name_; - const uint8_t * type_hash_; + const rosidl_type_hash_t type_hash_; /// The number of fields in the interface uint32_t member_count_; /// The size of the interface structure in memory From 2e17955a276835620a40e832ca956d2ec4e44423 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 20:26:43 -0800 Subject: [PATCH 36/65] Type hash cpp Signed-off-by: Emerson Knapp --- rosidl_generator_cpp/resource/msg__struct.hpp.em | 3 ++- .../message_introspection.hpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index c2bfbda56..4f2e9ef0d 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -34,6 +34,7 @@ msvc_common_macros = ('DELETE', 'ERROR', 'NO_ERROR') from collections import OrderedDict from rosidl_pycommon import convert_camel_case_to_lower_case_underscore includes = OrderedDict() +includes['rosidl_runtime_c/type_hash.h'] = [] for member in message.structure.members: type_ = member.type if isinstance(type_, AbstractNestedType): @@ -362,7 +363,7 @@ using @(message.structure.namespaced_type.name) = @(message_typename)_>; template -constexpr const uint8_t @(message.structure.namespaced_type.name)_::TYPE_VERSION_HASH[32]; +constexpr const rosidl_type_hash_t @(message.structure.namespaced_type.name)_::TYPE_VERSION_HASH; // constant definitions @[for c in message.constants]@ diff --git a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp index 1407d10fc..78e19089f 100644 --- a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp +++ b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp @@ -93,6 +93,7 @@ typedef struct ROSIDL_TYPESUPPORT_INTROSPECTION_CPP_PUBLIC MessageMembers_s const char * message_namespace_; /// The name of the interface, e.g. "Int16" const char * message_name_; + const rosidl_type_hash_t type_hash_; /// The number of fields in the interface uint32_t member_count_; From 3c57119581a75c0ab43035aab62828356d0ff234 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 21:00:26 -0800 Subject: [PATCH 37/65] Extern c for the helper functions, and extra includes Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 2 ++ .../resource/action__struct.hpp.em | 1 + .../include/rosidl_runtime_c/type_hash.h | 30 +++++++++++++++++-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index d797475d2..37fa91efb 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -33,6 +33,8 @@ extern "C" #include #include +#include "rosidl_runtime_c/type_hash.h" + @####################################################################### @# Handle message @####################################################################### diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index f34c54014..32da4d134 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -7,6 +7,7 @@ from rosidl_parser.definition import ACTION_GOAL_SUFFIX from rosidl_parser.definition import ACTION_RESULT_SERVICE_SUFFIX from rosidl_parser.definition import ACTION_RESULT_SUFFIX action_includes = ( + 'rosidl_runtime_c/type_hash.h', 'action_msgs/srv/cancel_goal.hpp', 'action_msgs/msg/goal_info.hpp', 'action_msgs/msg/goal_status_array.hpp', diff --git a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h index 17b414a27..f7dee2da4 100644 --- a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h +++ b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h @@ -25,16 +25,33 @@ #define ROSIDL_TYPE_HASH_VERSION_UNSET 0 #define ROSIDL_TYPE_HASH_SIZE RCUTILS_SHA256_BLOCK_SIZE +#ifdef __cplusplus +extern "C" +{ +#endif + +/// TODO(emersonknapp) +/** + * + */ typedef struct rosidl_type_hash_s { uint8_t version; uint8_t value[ROSIDL_TYPE_HASH_SIZE]; } rosidl_type_hash_t; +/// TODO(emersonknapp) +/** + * + */ ROSIDL_GENERATOR_C_PUBLIC rosidl_type_hash_t rosidl_get_zero_initialized_type_hash(void); +/// TODO(emersonknapp) +/** + * + */ ROSIDL_GENERATOR_C_PUBLIC rcutils_ret_t rosidl_stringify_type_hash( @@ -42,9 +59,16 @@ rosidl_stringify_type_hash( rcutils_allocator_t allocator, char ** output_string); -ROSIDL_GENERATOR_C_PUBLIC -rosidl_type_hash_t -rosidl_parse_type_hash_string(const char * type_hash_string, size_t length); +/// TODO(emersonknapp) +/** + * + */ +// ROSIDL_GENERATOR_C_PUBLIC +// rosidl_type_hash_t +// rosidl_parse_type_hash_string(const char * type_hash_string, size_t length); +#ifdef __cplusplus +} +#endif #endif // ROSIDL_RUNTIME_C__TYPE_HASH_H_ From d457eb4a549c5543dc6192b6e87f7431d873d9f2 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 21:35:57 -0800 Subject: [PATCH 38/65] Use rosidl def not sha256 Signed-off-by: Emerson Knapp --- rosidl_runtime_c/src/type_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index 2edffa0d3..464bca8f7 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -51,7 +51,7 @@ rosidl_stringify_type_hash( RCUTILS_SET_ERROR_MSG("Unable to allocate space for stringified type hash."); return RCUTILS_RET_BAD_ALLOC; } - for (size_t i = 0; i < RCUTILS_SHA256_BLOCK_SIZE; i++) { + for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { sprintf(local_output + (i * 2), "%02x", type_hash->value[i]); } fprintf(stderr, "%s", local_output); From 8fa63596804901b94640782cc63edb6e34cad26c Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 23:27:52 -0800 Subject: [PATCH 39/65] Parse type hash string utility Signed-off-by: Emerson Knapp --- .../include/rosidl_runtime_c/type_hash.h | 9 ++++-- rosidl_runtime_c/src/type_hash.c | 29 +++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h index f7dee2da4..0c4e9b6e7 100644 --- a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h +++ b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h @@ -63,9 +63,12 @@ rosidl_stringify_type_hash( /** * */ -// ROSIDL_GENERATOR_C_PUBLIC -// rosidl_type_hash_t -// rosidl_parse_type_hash_string(const char * type_hash_string, size_t length); +ROSIDL_GENERATOR_C_PUBLIC +rcutils_ret_t +rosidl_parse_type_hash_string( + const char * type_hash_string, + size_t length, + rosidl_type_hash_t * out); #ifdef __cplusplus } diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index 464bca8f7..5d53b45ef 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -45,6 +45,7 @@ rosidl_stringify_type_hash( // Hash representation will be simple hex string, two characters per byte const char * fmt = "RIHS%d_%064d"; + const size_t prefix_len = 7; // RIHS1_ char * local_output = rcutils_format_string(allocator, fmt, type_hash->version, 0); if (!local_output) { *output_string = NULL; @@ -52,10 +53,34 @@ rosidl_stringify_type_hash( return RCUTILS_RET_BAD_ALLOC; } for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { - sprintf(local_output + (i * 2), "%02x", type_hash->value[i]); + sprintf(local_output + prefix_len + (i * 2), "%02x", type_hash->value[i]); } - fprintf(stderr, "%s", local_output); *output_string = local_output; return RCUTILS_RET_OK; } + +rcutils_ret_t +rosidl_parse_type_hash_string( + const char * type_hash_string, + size_t data_length, + rosidl_type_hash_t * out) +{ + RCUTILS_CHECK_ARGUMENT_FOR_NULL(type_hash_string, RCUTILS_RET_INVALID_ARGUMENT); + RCUTILS_CHECK_ARGUMENT_FOR_NULL(out, RCUTILS_RET_INVALID_ARGUMENT); + static const size_t value_length = 64; // 32 bytes * 2 digit characters + + char hash_value_str[value_length + 1]; + int res = sscanf(type_hash_string, "RIHS%hhu_%64s", &out->version, hash_value_str); + if (res != 2) { + RCUTILS_SET_ERROR_MSG("Type hash data did not match expected format."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i += 1) { + if (sscanf(hash_value_str + (i * 2), "%2hhx", &out->value[i]) != 1) { + RCUTILS_SET_ERROR_MSG("Couldn't parse hex string of type hash value."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + } + return RCUTILS_RET_OK; +} From afbfb416cc3da35e6a0bfb0e35d593b713d81663 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 21 Feb 2023 23:47:15 -0800 Subject: [PATCH 40/65] Fix off-by-one stringifying Signed-off-by: Emerson Knapp --- rosidl_runtime_c/src/type_hash.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index 5d53b45ef..b951e6595 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -44,8 +44,8 @@ rosidl_stringify_type_hash( } // Hash representation will be simple hex string, two characters per byte - const char * fmt = "RIHS%d_%064d"; - const size_t prefix_len = 7; // RIHS1_ + const char * fmt = "RIHS%d_%64d"; + const size_t prefix_len = strlen("RIHS1_"); char * local_output = rcutils_format_string(allocator, fmt, type_hash->version, 0); if (!local_output) { *output_string = NULL; @@ -69,8 +69,7 @@ rosidl_parse_type_hash_string( RCUTILS_CHECK_ARGUMENT_FOR_NULL(type_hash_string, RCUTILS_RET_INVALID_ARGUMENT); RCUTILS_CHECK_ARGUMENT_FOR_NULL(out, RCUTILS_RET_INVALID_ARGUMENT); static const size_t value_length = 64; // 32 bytes * 2 digit characters - - char hash_value_str[value_length + 1]; + char hash_value_str[value_length]; int res = sscanf(type_hash_string, "RIHS%hhu_%64s", &out->version, hash_value_str); if (res != 2) { RCUTILS_SET_ERROR_MSG("Type hash data did not match expected format."); From ec3126a08e48134b71c84c537f351865687a6e87 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 22 Feb 2023 12:29:54 -0800 Subject: [PATCH 41/65] Code cleanup pass Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/msg__struct.h.em | 2 - .../resource/action__struct.hpp.em | 1 - .../resource/idl__struct.hpp.em | 1 + .../resource/msg__struct.hpp.em | 2 - rosidl_generator_type_hash/README.md | 2 +- .../bin/rosidl_generator_type_hash | 13 +++ ...erator_type_hash_generate_interfaces.cmake | 10 +- .../resource/TypeDescription.schema.json | 4 +- .../rosidl_generator_type_hash/__init__.py | 92 ++++++++++--------- rosidl_pycommon/rosidl_pycommon/__init__.py | 28 +++--- .../message_introspection.h | 2 +- .../service_introspection.h | 2 - .../message_introspection.hpp | 2 +- .../service_introspection.hpp | 2 - 14 files changed, 90 insertions(+), 73 deletions(-) diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index b242c6608..f1d7bfee5 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -28,7 +28,6 @@ from rosidl_generator_c import value_to_c @{ from collections import OrderedDict includes = OrderedDict() -includes.setdefault('rosidl_runtime_c/type_hash.h', []) for member in message.structure.members: if isinstance(member.type, AbstractSequence) and isinstance(member.type.value_type, BasicType): member_names = includes.setdefault( @@ -62,7 +61,6 @@ for member in message.structure.members: }@ @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< // Constants defined in the message @[for constant in message.constants]@ diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index 32da4d134..f34c54014 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -7,7 +7,6 @@ from rosidl_parser.definition import ACTION_GOAL_SUFFIX from rosidl_parser.definition import ACTION_RESULT_SERVICE_SUFFIX from rosidl_parser.definition import ACTION_RESULT_SUFFIX action_includes = ( - 'rosidl_runtime_c/type_hash.h', 'action_msgs/srv/cancel_goal.hpp', 'action_msgs/msg/goal_info.hpp', 'action_msgs/msg/goal_status_array.hpp', diff --git a/rosidl_generator_cpp/resource/idl__struct.hpp.em b/rosidl_generator_cpp/resource/idl__struct.hpp.em index becc1d439..6e1615ccc 100644 --- a/rosidl_generator_cpp/resource/idl__struct.hpp.em +++ b/rosidl_generator_cpp/resource/idl__struct.hpp.em @@ -29,6 +29,7 @@ include_directives = set() #include #include +#include "rosidl_runtime_c/type_hash.h" #include "rosidl_runtime_cpp/bounded_vector.hpp" #include "rosidl_runtime_cpp/message_initialization.hpp" diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index 4f2e9ef0d..1bf41e61f 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -34,7 +34,6 @@ msvc_common_macros = ('DELETE', 'ERROR', 'NO_ERROR') from collections import OrderedDict from rosidl_pycommon import convert_camel_case_to_lower_case_underscore includes = OrderedDict() -includes['rosidl_runtime_c/type_hash.h'] = [] for member in message.structure.members: type_ = member.type if isinstance(type_, AbstractNestedType): @@ -97,7 +96,6 @@ namespace @(ns) @[end for]@ // message struct - template struct @(message.structure.namespaced_type.name)_ { diff --git a/rosidl_generator_type_hash/README.md b/rosidl_generator_type_hash/README.md index 18cf4d73f..4279b4b9b 100644 --- a/rosidl_generator_type_hash/README.md +++ b/rosidl_generator_type_hash/README.md @@ -34,4 +34,4 @@ NOTE: `.json` contents must have all whitespace stripped before hashing. This pa **Schema**: [TypeDescriptionIn](./resource/TypeDescriptionIn.schema.json) -The `IndividualTypeDescription`-equivalent serialization, and a list of referenced `.in.json` files to be used when expanding to `.json`. This file is used by dependent interfaces, especially in dependent packages, when expanding to `.json` so IDL files must only be parsed a single time when running the generator on their own package. +The `IndividualTypeDescription`-equivalent serialization, and a list of referenced `.in.json` files to be used when expanding to `.json`. This file is used by dependent interfaces, especially in dependent packages, when expanding to `.json` so IDL files need only be parsed a single time when running the generator on their own package. diff --git a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash index 3bd8e5251..fbf92e874 100755 --- a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash +++ b/rosidl_generator_type_hash/bin/rosidl_generator_type_hash @@ -1,4 +1,17 @@ #!/usr/bin/env python3 +# Copyright 2023 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import os diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index 6c4185dc4..fa54f66fc 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -47,9 +47,10 @@ foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) endforeach() endforeach() -# Export __HASH_TUPLES variable for use by dependents +# Export __HASH_TUPLES variable for use by dependent generators set(${rosidl_generate_interfaces_TARGET}__HASH_TUPLES ${_generated_hash_tuples}) +# Validate that all dependencies exist set(target_dependencies "${rosidl_generator_type_hash_BIN}" ${rosidl_generator_type_hash_GENERATOR_FILES} @@ -70,6 +71,7 @@ rosidl_write_generator_arguments( INCLUDE_PATHS "${_dependency_paths}" ) +# Create custom command and target to generate the hash output set(_generated_files "${_generated_json};${_generated_json_in};${_generated_hash_files}") add_custom_command( COMMAND Python3::Interpreter @@ -85,7 +87,7 @@ add_custom_command( set(_target "${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_hash") add_custom_target(${_target} DEPENDS ${_generated_files}) -# # Make top level generation target depend on this generated library +# Make top level generation target depend on this generated library add_dependencies(${rosidl_generate_interfaces_TARGET} ${_target}) if(NOT rosidl_generate_interfaces_SKIP_INSTALL) @@ -96,8 +98,4 @@ if(NOT rosidl_generate_interfaces_SKIP_INSTALL) FILES ${_generated_file} DESTINATION "share/${PROJECT_NAME}/${_parent_folder}") endforeach() - - # TODO(emersonknapp) do I need this? - # Export modern CMake targets - # ament_export_targets(export_${rosidl_generate_interfaces_TARGET}${_target_suffix}) endif() diff --git a/rosidl_generator_type_hash/resource/TypeDescription.schema.json b/rosidl_generator_type_hash/resource/TypeDescription.schema.json index 2918c08ec..2d86acde0 100644 --- a/rosidl_generator_type_hash/resource/TypeDescription.schema.json +++ b/rosidl_generator_type_hash/resource/TypeDescription.schema.json @@ -4,11 +4,13 @@ "title": "TypeVersionHash", "description": "IndividualTypeDescription", "type": "object", + "$comment": "All whitespace must be excluded for consistent hashing.", "properties": { "type_description": {"$ref": "IndividualTypeDescription"}, "referenced_type_descriptions": { "type": "array", - "items": { "$ref": "IndividualTypeDescription" } + "items": { "$ref": "IndividualTypeDescription" }, + "$comment": "Referenced type descriptions must be alphabetized." } }, "required": ["type_description", "referenced_type_descriptions"] diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 1367f3b91..1fecab737 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -21,6 +21,9 @@ from rosidl_parser import definition from rosidl_parser.parser import parse_idl_file +# ROS Interface Hashing Standard, per REP-2011 +RIHS_VERSION = 1 + def generate_type_hash(generator_arguments_file: str) -> List[str]: with open(generator_arguments_file, 'r') as f: @@ -30,7 +33,7 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: idl_tuples = args['idl_tuples'] include_paths = args.get('include_paths', []) - # Lookup for directory containing pregenerated .in.json files + # Lookup for directory containing dependency .in.json files include_map = { package_name: output_dir } @@ -75,41 +78,40 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: # This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg -# TODO(emersonknapp) -# There is no FieldType.msg definition for the following rosidl_parser.definition types -# * SIGNED_NONEXPLICIT_INTEGER_TYPES = short, long, long long -# * UNSIGNED_NONEXPLICIT_INTEGER_TYPES = unsigned short, unsigned long, unsigned long long +# NOTE: Nonexplicit integer types are not defined in FieldType (short, long, long long). +# If a ROS IDL uses these, this generator will throw a KeyError. FIELD_TYPES = { - 'nested_type': 0, - 'int8': 1, - 'uint8': 2, - 'int16': 3, - 'uint16': 4, - 'int32': 5, - 'uint32': 6, - 'int64': 7, - 'uint64': 8, - 'float': 9, - 'double': 10, - 'long double': 11, - 'char': 12, - 'wchar': 13, - 'boolean': 14, - 'octet': 15, # byte - definition.UnboundedString: 16, - definition.UnboundedWString: 17, - # TODO(emersonknapp) - # there is no rosidl_parser.definition type FixedString - # FIXED_STRING = 18 - # FIXED_WSTRING = 19 - definition.BoundedString: 20, - definition.BoundedWString: 21, + # 0 reserved for "Not set" + 'nested_type': 1, + 'int8': 2, + 'uint8': 3, + 'int16': 4, + 'uint16': 5, + 'int32': 6, + 'uint32': 7, + 'int64': 8, + 'uint64': 9, + 'float': 10, + 'double': 11, + 'long double': 12, + 'char': 13, + 'wchar': 14, + 'boolean': 15, + 'octet': 16, # byte + definition.UnboundedString: 17, + definition.UnboundedWString: 18, + # NOTE: rosidl_parser does not define fixed string types + # FIXED_STRING: 19 + # FIXED_WSTRING: 20 + definition.BoundedString: 21, + definition.BoundedWString: 22, } +FIELD_TYPE_BLOCK_SIZE = 48 NESTED_FIELD_TYPE_OFFSETS = { - definition.Array: 32, - definition.BoundedSequence: 32 * 2, - definition.UnboundedSequence: 32 * 3, + definition.Array: FIELD_TYPE_BLOCK_SIZE, + definition.BoundedSequence: FIELD_TYPE_BLOCK_SIZE * 2, + definition.UnboundedSequence: FIELD_TYPE_BLOCK_SIZE * 3, } @@ -171,15 +173,14 @@ def serialize_individual_type_description( class InterfaceHasher: + """Contains context about subinterfaces for a given interface description.""" @classmethod def from_idl(cls, idl: definition.IdlFile): for el in idl.content.elements: - if isinstance(el, definition.Message): - return InterfaceHasher(el) - elif isinstance(el, definition.Service): - return InterfaceHasher(el) - elif isinstance(el, definition.Action): + if any(isinstance(el, type_) for type_ in [ + definition.Message, definition.Service, definition.Action + ]): return InterfaceHasher(el) raise Exception('No interface found in IDL') @@ -248,6 +249,7 @@ def __init__(self, interface): } def write_json_in(self, output_dir) -> List[str]: + """Return list of written files.""" generated_files = [] for key, val in self.subinterfaces.items(): generated_files += val.write_json_in(output_dir) @@ -259,6 +261,7 @@ def write_json_in(self, output_dir) -> List[str]: return generated_files + [str(json_path)] def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: + """Return list of written files.""" generated_files = [] for key, val in self.subinterfaces.items(): generated_files += val.write_json_out(output_dir, includes_map) @@ -281,6 +284,7 @@ def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: loaded_includes[process_include] = include_json['type_description'] pending_includes.extend(include_json['includes']) + # Sort included type descriptions alphabetically by type name self.json_out = { 'type_description': self.json_in['type_description'], 'referenced_type_descriptions': sorted( @@ -289,25 +293,27 @@ def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: json_path = output_dir / self.rel_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json.dumps(self.json_out)) + json_file.write(json.dumps(self.json_out)) # NOTE: no whitespace! return generated_files + [str(json_path)] - def calculate_hash(self) -> dict: - json_out_repr = json.dumps(self.json_out) + def _calculate_hash_tree(self) -> dict: + prefix = f'RIHS{RIHS_VERSION}_' + json_out_repr = json.dumps(self.json_out) # NOTE: no whitespace! sha = hashlib.sha256() sha.update(json_out_repr.encode('utf-8')) - type_hash = sha.hexdigest() + type_hash = prefix + sha.hexdigest() type_hash_infos = { self.interface_type: type_hash, } for key, val in self.subinterfaces.items(): - type_hash_infos[key] = val.calculate_hash() + type_hash_infos[key] = val._calculate_hash_tree() return type_hash_infos def write_hash(self, output_dir: Path) -> List[str]: - type_hash = self.calculate_hash() + """Return list of written files.""" + type_hash = self._calculate_hash_tree() hash_path = output_dir / self.rel_path.with_suffix('.sha256.json') with hash_path.open('w', encoding='utf-8') as hash_file: hash_file.write(json.dumps(type_hash, indent=2)) diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 24a8d5733..02f68afc6 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -222,15 +222,21 @@ def _expand_template(template_name, **kwargs): def _expand_type_hash(variable_name, hash_string, indent=0): - """ - Generate a rosidl_type_hash_t instance with 8 bytes per line for readability. - """ - ind_str = ' ' * indent - RIHS_VERSION = 1 # TODO(emersonknapp) where should I get this from.... IS IT IN THE HEX_STRING? - result = f'rosidl_type_hash_t {variable_name} = {{{RIHS_VERSION}, {{' - for i in range(32): - if i % 8 == 0: - result += f'\n{ind_str} ' - result += f'0x{hash_string[i*2:i*2+2]}, ' - result += f'\n{ind_str}}}}};\n' + """Generate empy for rosidl_type_hash_t instance with 8 bytes per line for readability.""" + hash_length = 32 + bytes_per_line = 8 + + indent_str = ' ' * indent + pattern = re.compile('RIHS(\d+)_([0-9a-f]{64})') + match = pattern.match(hash_string) + if not match: + raise Exception('Type hash string does not match expected RIHS format') + version, value = match.group(1, 2) + + result = f'rosidl_type_hash_t {variable_name} = {{{version}, {{' + for i in range(hash_length): + if i % bytes_per_line == 0: + result += f'\n{indent_str} ' + result += f'0x{value[i * 2:i * 2 + 2]}, ' + result += f'\n{indent_str}}}}};\n' return result diff --git a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h index e105a8e1b..12fc4811a 100644 --- a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h +++ b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/message_introspection.h @@ -86,7 +86,7 @@ typedef struct rosidl_typesupport_introspection_c__MessageMembers_s const char * message_namespace_; /// The name of the interface, e.g. "Int16" const char * message_name_; - /// + /// Hashed value of the interface description const rosidl_type_hash_t type_hash_; /// The number of fields in the interface uint32_t member_count_; diff --git a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h index 007569800..ea57dffd6 100644 --- a/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h +++ b/rosidl_typesupport_introspection_c/include/rosidl_typesupport_introspection_c/service_introspection.h @@ -32,8 +32,6 @@ typedef struct rosidl_typesupport_introspection_c__ServiceMembers_s const char * service_namespace_; /// The name of the service, e.g. "AddTwoInts" const char * service_name_; - /// TODO(emersonknapp) - // const uint8_t * type_hash_; /// A pointer to the introspection information structure for the request interface. const rosidl_typesupport_introspection_c__MessageMembers * request_members_; /// A pointer to the introspection information structure for the response interface. diff --git a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp index 78e19089f..cc60501f8 100644 --- a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp +++ b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/message_introspection.hpp @@ -93,7 +93,7 @@ typedef struct ROSIDL_TYPESUPPORT_INTROSPECTION_CPP_PUBLIC MessageMembers_s const char * message_namespace_; /// The name of the interface, e.g. "Int16" const char * message_name_; - + /// Hashed value of the interface description const rosidl_type_hash_t type_hash_; /// The number of fields in the interface uint32_t member_count_; diff --git a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp index 187e29fd5..cad7e04e8 100644 --- a/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp +++ b/rosidl_typesupport_introspection_cpp/include/rosidl_typesupport_introspection_cpp/service_introspection.hpp @@ -35,8 +35,6 @@ typedef struct ServiceMembers_s const char * service_namespace_; /// The name of the service, e.g. "AddTwoInts" const char * service_name_; - /// TODO(emersonknapp) - // const uint8_t * service_type_hash_; /// A pointer to the introspection information structure for the request interface. const MessageMembers * request_members_; /// A pointer to the introspection information structure for the response interface. From 226a0d184074a56469249e92da7ca7b6925400ff Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 22 Feb 2023 14:08:21 -0800 Subject: [PATCH 42/65] Add docs and tests for rosidl_type_hash Signed-off-by: Emerson Knapp --- rosidl_runtime_c/CMakeLists.txt | 5 + .../include/rosidl_runtime_c/type_hash.h | 29 +++--- rosidl_runtime_c/src/type_hash.c | 20 ++-- rosidl_runtime_c/test/test_type_hash.cpp | 96 +++++++++++++++++++ 4 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 rosidl_runtime_c/test/test_type_hash.cpp diff --git a/rosidl_runtime_c/CMakeLists.txt b/rosidl_runtime_c/CMakeLists.txt index e6b162e8e..5da904f1c 100644 --- a/rosidl_runtime_c/CMakeLists.txt +++ b/rosidl_runtime_c/CMakeLists.txt @@ -99,6 +99,11 @@ if(BUILD_TESTING) target_compile_definitions(test_string_functions PUBLIC RCUTILS_ENABLE_FAULT_INJECTION) endif() + ament_add_gtest(test_type_hash test/test_type_hash.cpp) + if(TARGET test_type_hash) + target_link_libraries(test_type_hash ${PROJECT_NAME}) + endif() + ament_add_gtest(test_u16string_functions test/test_u16string_functions.cpp) if(TARGET test_u16string_functions) target_link_libraries(test_u16string_functions ${PROJECT_NAME}) diff --git a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h index 0c4e9b6e7..7fa66893a 100644 --- a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h +++ b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h @@ -30,27 +30,32 @@ extern "C" { #endif -/// TODO(emersonknapp) -/** - * - */ +/// A ROS 2 interface type hash per REP-2011 RIHS standard. typedef struct rosidl_type_hash_s { uint8_t version; uint8_t value[ROSIDL_TYPE_HASH_SIZE]; } rosidl_type_hash_t; -/// TODO(emersonknapp) +/// Get a new zero-initialized type hash structure. /** - * + * Note that the version equals ROSIDL_TYPE_HASH_VERSION_UNSET. */ ROSIDL_GENERATOR_C_PUBLIC rosidl_type_hash_t rosidl_get_zero_initialized_type_hash(void); -/// TODO(emersonknapp) +/// Convert type hash to a standardized string representation. /** + * Follows format RIHS{version}_{value}. * + * \param[in] type_hash Type hash to convert to string + * \param[in] allocator Allocator to use for allocating string space + * \param[out] output_string Handle to a pointer that will be set + * to the newly allocated null-terminated string representation. + * \return RCUTILS_RET_INVALID_ARGUMENT if any pointer arguments are null or allocator invalid + * \return RCUTILS_RET_BAD_ALLOC if space could not be allocated for resulting string + * \return RCUTILS_RET_OK otherwise */ ROSIDL_GENERATOR_C_PUBLIC rcutils_ret_t @@ -59,16 +64,18 @@ rosidl_stringify_type_hash( rcutils_allocator_t allocator, char ** output_string); -/// TODO(emersonknapp) +/// Parse a stringified type hash to a struct. /** - * + * \param[in] type_hash_string Null-terminated string with the hash representation + * \param[out] hash_out Preallocated structure to be filled with parsed hash information. + * \return RCTUILS_RET_INVALID_ARGUMENT on any null pointer argumunts, or malformed hash string. + * \return RCUTILS_RET_OK otherwise */ ROSIDL_GENERATOR_C_PUBLIC rcutils_ret_t rosidl_parse_type_hash_string( const char * type_hash_string, - size_t length, - rosidl_type_hash_t * out); + rosidl_type_hash_t * hash_out); #ifdef __cplusplus } diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index b951e6595..3900a9537 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "rosidl_runtime_c/type_hash.h" #include "rcutils/error_handling.h" @@ -63,20 +65,26 @@ rosidl_stringify_type_hash( rcutils_ret_t rosidl_parse_type_hash_string( const char * type_hash_string, - size_t data_length, - rosidl_type_hash_t * out) + rosidl_type_hash_t * hash_out) { RCUTILS_CHECK_ARGUMENT_FOR_NULL(type_hash_string, RCUTILS_RET_INVALID_ARGUMENT); - RCUTILS_CHECK_ARGUMENT_FOR_NULL(out, RCUTILS_RET_INVALID_ARGUMENT); + RCUTILS_CHECK_ARGUMENT_FOR_NULL(hash_out, RCUTILS_RET_INVALID_ARGUMENT); static const size_t value_length = 64; // 32 bytes * 2 digit characters - char hash_value_str[value_length]; - int res = sscanf(type_hash_string, "RIHS%hhu_%64s", &out->version, hash_value_str); + char hash_value_str[value_length + 1]; + hash_value_str[value_length] = '\0'; + int res = sscanf(type_hash_string, "RIHS%hhu_%64s", &hash_out->version, hash_value_str); if (res != 2) { RCUTILS_SET_ERROR_MSG("Type hash data did not match expected format."); return RCUTILS_RET_INVALID_ARGUMENT; } + size_t version_digits = log10(hash_out->version) + 1; + size_t prefix_fixed_len = strlen("RIHS_"); + if (strlen(type_hash_string) > value_length + prefix_fixed_len + version_digits) { + RCUTILS_SET_ERROR_MSG("Hash value too long."); + return RCUTILS_RET_INVALID_ARGUMENT; + } for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i += 1) { - if (sscanf(hash_value_str + (i * 2), "%2hhx", &out->value[i]) != 1) { + if (sscanf(hash_value_str + (i * 2), "%2hhx", &hash_out->value[i]) != 1) { RCUTILS_SET_ERROR_MSG("Couldn't parse hex string of type hash value."); return RCUTILS_RET_INVALID_ARGUMENT; } diff --git a/rosidl_runtime_c/test/test_type_hash.cpp b/rosidl_runtime_c/test/test_type_hash.cpp new file mode 100644 index 000000000..895e066d6 --- /dev/null +++ b/rosidl_runtime_c/test/test_type_hash.cpp @@ -0,0 +1,96 @@ +// Copyright 2023 Open Source Robotics Foundation, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" + +#include "rcutils/error_handling.h" +#include "rosidl_runtime_c/type_hash.h" + +TEST(type_hash, init_zero_hash) { + auto hash = rosidl_get_zero_initialized_type_hash(); + EXPECT_EQ(hash.version, 0); + for (size_t i = 0; i < sizeof(hash.value); i++) { + EXPECT_EQ(hash.value[i], 0); + } +} + +TEST(type_hash, stringify_basic) { + const std::string expected = + "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + hash.version = 1; + for (size_t i = 0; i < sizeof(hash.value); i++) { + hash.value[i] = (i % 0x10) * 0x10 + (i % 0x10); + } + auto allocator = rcutils_get_default_allocator(); + char * hash_string = nullptr; + ASSERT_EQ(RCUTILS_RET_OK, rosidl_stringify_type_hash(&hash, allocator, &hash_string)); + ASSERT_TRUE(hash_string); + + std::string cpp_str(hash_string); + EXPECT_EQ(expected, hash_string); +} + +TEST(type_hash, parse_basic) { + const std::string test_value = + "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + ASSERT_EQ(RCUTILS_RET_OK, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + EXPECT_EQ(1, hash.version); + for (size_t i = 0; i < sizeof(hash.value); i++) { + size_t expected_value = (i % 0x10) * 0x10 + (i % 0x10); + EXPECT_EQ(expected_value, hash.value[i]); + } +} + +TEST(type_hash, parse_bad_prefix) { + const std::string test_value = + "RRRR1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + rcutils_reset_error(); +} + +TEST(type_hash, parse_no_version) { + const std::string test_value = + "RIHS_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + rcutils_reset_error(); +} + +TEST(type_hash, parse_too_short) { + const std::string test_value = + "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddee"; + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + rcutils_reset_error(); +} + +TEST(type_hash, parse_too_long) { + const std::string test_value = + "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff00"; + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + rcutils_reset_error(); +} + +TEST(type_hash, parse_multidigit_version) { + const std::string test_value = + "RIHS213_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + ASSERT_EQ(RCUTILS_RET_OK, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + EXPECT_EQ(213, hash.version); +} From b2a413276ccfebd8c26bca2a8a199aa6de0751a5 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 22 Feb 2023 14:30:36 -0800 Subject: [PATCH 43/65] Clean up schemata Signed-off-by: Emerson Knapp --- .../IndividualTypeDescription.schema.json | 13 ++++++++----- .../resource/TypeDescription.schema.json | 15 ++++++++------- .../resource/TypeDescriptionIn.schema.json | 13 +++++++------ .../resource/TypeVersionHash.schema.json | 11 +++++++---- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json b/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json index 24149d251..5e82291fd 100644 --- a/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json +++ b/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "IndividualTypeDescription.jsonschema", - "title": "TypeVersionHash", - "description": "IndividualTypeDescription", + "$id": "IndividualTypeDescription.schema.json", + "title": "IndividualTypeDescription", + "description": "type_description_interfaces/msg/IndividualTypeDescription.msg JSON representation", "type": "object", "properties": { "type_name": {"type": "string", "maxLength": 255}, @@ -14,6 +14,7 @@ } }, "required": ["type_name", "fields"], + "additionalProperties": false, "$defs": { "Field": { "type": "object", @@ -21,7 +22,8 @@ "name": {"type": "string"}, "type": {"$ref": "#/$defs/FieldType"} }, - "required": ["name", "type"] + "required": ["name", "type"], + "additionalProperties": false }, "FieldType": { "type": "object", @@ -31,7 +33,8 @@ "string_length": {"type": "integer", "minimum": 0}, "nested_type_name": {"type": "string", "maxLength": 255} }, - "required": ["type_id", "length", "string_length", "nested_type_name"] + "required": ["type_id", "length", "string_length", "nested_type_name"], + "additionalProperties": false } } } diff --git a/rosidl_generator_type_hash/resource/TypeDescription.schema.json b/rosidl_generator_type_hash/resource/TypeDescription.schema.json index 2d86acde0..c4ec81a40 100644 --- a/rosidl_generator_type_hash/resource/TypeDescription.schema.json +++ b/rosidl_generator_type_hash/resource/TypeDescription.schema.json @@ -1,17 +1,18 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "TypeDescription.schema.json", - "title": "TypeVersionHash", - "description": "IndividualTypeDescription", + "title": "TypeDescription", + "description": "type_description_interfaces/msg/TypeDescription.msg JSON representation", "type": "object", - "$comment": "All whitespace must be excluded for consistent hashing.", + "$comment": "All whitespace must be excluded for consistent hashing, which this schema cannot enforce.", "properties": { - "type_description": {"$ref": "IndividualTypeDescription"}, + "type_description": {"$ref": "file:IndividualTypeDescription.schema.json"}, "referenced_type_descriptions": { + "$comment": "Referenced type descriptions must be alphabetized, which this schema cannot enforce.", "type": "array", - "items": { "$ref": "IndividualTypeDescription" }, - "$comment": "Referenced type descriptions must be alphabetized." + "items": { "$ref": "file:IndividualTypeDescription.schema.json" } } }, - "required": ["type_description", "referenced_type_descriptions"] + "required": ["type_description", "referenced_type_descriptions"], + "additionalProperties": false } diff --git a/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json b/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json index 9c0bd2034..8be941749 100644 --- a/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json +++ b/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json @@ -1,15 +1,16 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "TypeDescription.schema.json", - "title": "TypeVersionHash", - "description": "IndividualTypeDescription", + "$id": "TypeDescriptionIn.schema.json", + "title": "TypeDescriptionIn", + "description": "Input for TypeDescription expansion, providing include paths for references", "type": "object", "properties": { - "type_description": {"$ref": "IndividualTypeDescription"}, - "referenced_type_descriptions": { + "type_description": {"$ref": "file:IndividualTypeDescription.schema.json"}, + "includes": { "type": "array", "items": { "type": "string" } } }, - "required": ["type_description", "referenced_type_descriptions"] + "required": ["type_description", "includes"], + "additionalProperties": false } diff --git a/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json b/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json index 4fa19c8ba..fe88c3548 100644 --- a/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json +++ b/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "TypeVersionHash.jsonschema", + "$id": "TypeVersionHash.schema.json", "title": "TypeVersionHash", "description": "A tree of hashes (and sub-hashes) for ROS 2 interfaces", "type": "object", @@ -10,7 +10,8 @@ "properties": { "message": { "type": "string" } }, - "required": [ "message" ] + "required": [ "message" ], + "additionalProperties": false }, "service": { "type": "object", @@ -25,7 +26,8 @@ "request_message", "response_message", "event_message" - ] + ], + "additionalProperties": false }, "action": { "type": "object", @@ -46,7 +48,8 @@ "send_goal_service", "get_result_service", "feedback_message" - ] + ], + "additionalProperties": false } }, "oneOf": [ From 9dbd2c657da8672d77e74d7589ebb98d0cbb232c Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 22 Feb 2023 16:35:39 -0800 Subject: [PATCH 44/65] Add schema checking test Signed-off-by: Emerson Knapp --- rosidl_generator_tests/CMakeLists.txt | 7 ++ rosidl_generator_tests/package.xml | 4 ++ .../test_type_hash.py | 67 +++++++++++++++++++ rosidl_generator_type_hash/CMakeLists.txt | 2 +- rosidl_pycommon/rosidl_pycommon/__init__.py | 6 +- rosidl_runtime_c/CMakeLists.txt | 1 + 6 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py diff --git a/rosidl_generator_tests/CMakeLists.txt b/rosidl_generator_tests/CMakeLists.txt index b428bfc7b..4f6ffc44d 100644 --- a/rosidl_generator_tests/CMakeLists.txt +++ b/rosidl_generator_tests/CMakeLists.txt @@ -19,9 +19,11 @@ find_package(ament_cmake REQUIRED) if(BUILD_TESTING) find_package(ament_cmake_gtest REQUIRED) + find_package(ament_cmake_pytest REQUIRED) find_package(ament_lint_auto REQUIRED) find_package(rosidl_cmake REQUIRED) find_package(rosidl_generator_cpp REQUIRED) + find_package(rosidl_generator_type_hash REQUIRED) find_package(rosidl_runtime_c REQUIRED) find_package(rosidl_runtime_cpp REQUIRED) find_package(test_interface_files REQUIRED) @@ -30,6 +32,7 @@ if(BUILD_TESTING) rosidl_generate_interfaces(${PROJECT_NAME} ${test_interface_files_MSG_FILES} ${test_interface_files_SRV_FILES} + ${test_interface_files_ACTION_FILES} ADD_LINTER_TESTS SKIP_INSTALL ) @@ -108,6 +111,10 @@ if(BUILD_TESTING) ${c_generator_target} rosidl_runtime_c::rosidl_runtime_c ) + + ament_add_pytest_test(test_hash_generator test/rosidl_generator_type_hash + ENV GENERATED_TEST_FILE_DIR=${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash/${PROJECT_NAME} + ) endif() ament_package() diff --git a/rosidl_generator_tests/package.xml b/rosidl_generator_tests/package.xml index 6b94bc6cc..be0f1d683 100644 --- a/rosidl_generator_tests/package.xml +++ b/rosidl_generator_tests/package.xml @@ -18,12 +18,16 @@ ament_cmake + action_msgs ament_cmake_gtest ament_lint_auto ament_lint_common + ament_index_python + python3-jsonschema rosidl_cmake rosidl_generator_c rosidl_generator_cpp + rosidl_generator_type_hash rosidl_runtime_c rosidl_runtime_cpp test_interface_files diff --git a/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py b/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py new file mode 100644 index 000000000..653e5b0b0 --- /dev/null +++ b/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py @@ -0,0 +1,67 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from pathlib import Path + +from ament_index_python import get_package_share_directory +import jsonschema + + +def test_type_hash(): + """Test all rosidl_generator_type_hash output files against defined schemas.""" + schema_dir = Path(get_package_share_directory('rosidl_generator_type_hash')) / 'resource' + resolver = jsonschema.validators.RefResolver( + base_uri=f'{schema_dir.as_uri()}/', + referrer=True, + ) + + generated_files_dir = Path(os.environ['GENERATED_TEST_FILE_DIR']) + validated_sha256 = 0 + validated_json_in = 0 + validated_json = 0 + for namespace in generated_files_dir.iterdir(): + for p in namespace.iterdir(): + assert p.is_file() + assert p.suffix == '.json' + with p.open('r') as f: + instance = json.load(f) + subsuffix = p.with_suffix('').suffix + if subsuffix == '.sha256': + jsonschema.validate( + instance=instance, + schema={'$ref': 'TypeVersionHash.schema.json'}, + resolver=resolver, + ) + validated_sha256 += 1 + elif subsuffix == '.in': + jsonschema.validate( + instance=instance, + schema={'$ref': 'TypeDescriptionIn.schema.json'}, + resolver=resolver, + ) + validated_json_in += 1 + elif subsuffix == '': + jsonschema.validate( + instance=instance, + schema={'$ref': 'TypeDescription.schema.json'}, + resolver=resolver, + ) + validated_json += 1 + else: + assert False, 'Unknown file type to validate' + assert validated_sha256, 'Needed to validate at least one of each type of file.' + assert validated_json_in, 'Needed to validate at least one of each type of file.' + assert validated_json, 'Needed to validate at least one of each type of file.' diff --git a/rosidl_generator_type_hash/CMakeLists.txt b/rosidl_generator_type_hash/CMakeLists.txt index 45061e95e..7f1a0c29b 100644 --- a/rosidl_generator_type_hash/CMakeLists.txt +++ b/rosidl_generator_type_hash/CMakeLists.txt @@ -22,6 +22,6 @@ install( DESTINATION lib/${PROJECT_NAME} ) install( - DIRECTORY cmake + DIRECTORY cmake resource DESTINATION share/${PROJECT_NAME} ) diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 02f68afc6..bcfbd454a 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -226,7 +226,7 @@ def _expand_type_hash(variable_name, hash_string, indent=0): hash_length = 32 bytes_per_line = 8 - indent_str = ' ' * indent + indent_str = ' ' * (indent + 2) pattern = re.compile('RIHS(\d+)_([0-9a-f]{64})') match = pattern.match(hash_string) if not match: @@ -237,6 +237,8 @@ def _expand_type_hash(variable_name, hash_string, indent=0): for i in range(hash_length): if i % bytes_per_line == 0: result += f'\n{indent_str} ' - result += f'0x{value[i * 2:i * 2 + 2]}, ' + result += f'0x{value[i * 2:i * 2 + 2]},' + if i % bytes_per_line != bytes_per_line - 1: + result += ' ' result += f'\n{indent_str}}}}};\n' return result diff --git a/rosidl_runtime_c/CMakeLists.txt b/rosidl_runtime_c/CMakeLists.txt index 5da904f1c..202a76f7e 100644 --- a/rosidl_runtime_c/CMakeLists.txt +++ b/rosidl_runtime_c/CMakeLists.txt @@ -30,6 +30,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC ament_target_dependencies(${PROJECT_NAME} "rcutils" "rosidl_typesupport_interface") +target_link_libraries(${PROJECT_NAME} m) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang") set_target_properties(${PROJECT_NAME} PROPERTIES COMPILE_OPTIONS -Wall -Wextra -Wpedantic) From ec45a34d316be2d00f78ead776fece53a1f5a90d Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 22 Feb 2023 16:52:17 -0800 Subject: [PATCH 45/65] Fix tests Signed-off-by: Emerson Knapp --- rosidl_pycommon/rosidl_pycommon/__init__.py | 2 +- rosidl_runtime_c/src/type_hash.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index bcfbd454a..419907352 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -227,7 +227,7 @@ def _expand_type_hash(variable_name, hash_string, indent=0): bytes_per_line = 8 indent_str = ' ' * (indent + 2) - pattern = re.compile('RIHS(\d+)_([0-9a-f]{64})') + pattern = re.compile(r'RIHS(\d+)_([0-9a-f]{64})') match = pattern.match(hash_string) if not match: raise Exception('Type hash string does not match expected RIHS format') diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index 3900a9537..276cfa43b 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -55,7 +55,7 @@ rosidl_stringify_type_hash( return RCUTILS_RET_BAD_ALLOC; } for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { - sprintf(local_output + prefix_len + (i * 2), "%02x", type_hash->value[i]); + snprintf(local_output + prefix_len + (i * 2), 3, "%02x", type_hash->value[i]); // NOLINT } *output_string = local_output; @@ -69,9 +69,9 @@ rosidl_parse_type_hash_string( { RCUTILS_CHECK_ARGUMENT_FOR_NULL(type_hash_string, RCUTILS_RET_INVALID_ARGUMENT); RCUTILS_CHECK_ARGUMENT_FOR_NULL(hash_out, RCUTILS_RET_INVALID_ARGUMENT); - static const size_t value_length = 64; // 32 bytes * 2 digit characters - char hash_value_str[value_length + 1]; - hash_value_str[value_length] = '\0'; + static const size_t kValueStringSize = 65; // 32 bytes * 2 digit characters + null-terminator + char hash_value_str[kValueStringSize]; + hash_value_str[kValueStringSize - 1] = '\0'; int res = sscanf(type_hash_string, "RIHS%hhu_%64s", &hash_out->version, hash_value_str); if (res != 2) { RCUTILS_SET_ERROR_MSG("Type hash data did not match expected format."); @@ -79,7 +79,7 @@ rosidl_parse_type_hash_string( } size_t version_digits = log10(hash_out->version) + 1; size_t prefix_fixed_len = strlen("RIHS_"); - if (strlen(type_hash_string) > value_length + prefix_fixed_len + version_digits) { + if (strlen(type_hash_string) > kValueStringSize - 1 + prefix_fixed_len + version_digits) { RCUTILS_SET_ERROR_MSG("Hash value too long."); return RCUTILS_RET_INVALID_ARGUMENT; } From 166e16ead2e5cd3de23a030f70c06be2082b045f Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 23 Feb 2023 16:57:29 -0800 Subject: [PATCH 46/65] Fix length/string_length field serialization and add a test Signed-off-by: Emerson Knapp --- rosidl_generator_type_hash/CMakeLists.txt | 2 + .../rosidl_generator_type_hash/__init__.py | 9 +++- .../test/test_serializers.py | 54 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 rosidl_generator_type_hash/test/test_serializers.py diff --git a/rosidl_generator_type_hash/CMakeLists.txt b/rosidl_generator_type_hash/CMakeLists.txt index 7f1a0c29b..5e95030be 100644 --- a/rosidl_generator_type_hash/CMakeLists.txt +++ b/rosidl_generator_type_hash/CMakeLists.txt @@ -10,7 +10,9 @@ ament_python_install_package(${PROJECT_NAME}) if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) + find_package(ament_cmake_pytest REQUIRED) ament_lint_auto_find_test_dependencies() + ament_add_pytest_test(pytest_type_hash_generator test) endif() ament_package( diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 1fecab737..0ec58ec93 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -130,6 +130,11 @@ def serialize_field_type(ftype: definition.AbstractType) -> dict: elif isinstance(ftype, definition.AbstractNestedType): type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] value_type = ftype.value_type + if ftype.has_maximum_size(): + try: + result['length'] = ftype.maximum_size + except AttributeError: + result['length'] = ftype.size else: raise Exception('Unable to translate field type', ftype) @@ -140,9 +145,9 @@ def serialize_field_type(ftype: definition.AbstractType) -> dict: result['type_id'] = FIELD_TYPES[type(value_type)] + type_id_offset if value_type.has_maximum_size(): try: - result['length'] = value_type.maximum_size + result['string_length'] = value_type.maximum_size except AttributeError: - result['length'] = value_type.size + result['string_length'] = value_type.size elif isinstance(value_type, definition.NamespacedType): result['type_id'] = type_id_offset result['nested_type_name'] = '/'.join(value_type.namespaced_name()) diff --git a/rosidl_generator_type_hash/test/test_serializers.py b/rosidl_generator_type_hash/test/test_serializers.py new file mode 100644 index 000000000..c52bd930a --- /dev/null +++ b/rosidl_generator_type_hash/test/test_serializers.py @@ -0,0 +1,54 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rosidl_generator_type_hash import serialize_field_type + +from rosidl_parser import definition + + +def test_field_type_serializer(): + # Sanity check for the more complex length/string_length types and nesting + string_limit = 12 + array_size = 22 + test_type = definition.Array(definition.BoundedString(string_limit), array_size) + expected = { + 'type_id': 69, + 'length': array_size, + 'string_length': string_limit, + 'nested_type_name': '', + } + + result = serialize_field_type(test_type) + assert result == expected + + bounded_sequence_limit = 32 + test_type = definition.BoundedSequence(definition.UnboundedString(), bounded_sequence_limit) + expected = { + 'type_id': 113, + 'length': bounded_sequence_limit, + 'string_length': 0, + 'nested_type_name': '', + } + result = serialize_field_type(test_type) + assert result == expected + + test_type = definition.BoundedWString(string_limit) + expected = { + 'type_id': 22, + 'length': 0, + 'string_length': string_limit, + 'nested_type_name': '', + } + result = serialize_field_type(test_type) + assert result == expected From 705aed02cc811b2a6633ac1eaf52f9b6f2edc4d4 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 23 Feb 2023 17:05:56 -0800 Subject: [PATCH 47/65] Address review comments Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 0ec58ec93..9cb14f90d 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -187,7 +187,7 @@ def from_idl(cls, idl: definition.IdlFile): definition.Message, definition.Service, definition.Action ]): return InterfaceHasher(el) - raise Exception('No interface found in IDL') + raise ValueError('No interface found in IDL') def __init__(self, interface): self.interface = interface @@ -265,6 +265,18 @@ def write_json_in(self, output_dir) -> List[str]: json_file.write(json.dumps(self.json_in, indent=2)) return generated_files + [str(json_path)] + def _hashable_repr(self) -> str: + return json.dumps( + self.json_out, + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=False, + indent=None, + separators=(',', ': '), + sort_keys=False + ).encode('utf-8') + def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: """Return list of written files.""" generated_files = [] @@ -298,14 +310,13 @@ def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: json_path = output_dir / self.rel_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json.dumps(self.json_out)) # NOTE: no whitespace! + json_file.write(self._hashable_repr()) return generated_files + [str(json_path)] def _calculate_hash_tree(self) -> dict: prefix = f'RIHS{RIHS_VERSION}_' - json_out_repr = json.dumps(self.json_out) # NOTE: no whitespace! sha = hashlib.sha256() - sha.update(json_out_repr.encode('utf-8')) + sha.update(self._hashable_repr()) type_hash = prefix + sha.hexdigest() type_hash_infos = { From 82ed9b4f2153fc3238c44a7a2aadb4b718e78055 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 23 Feb 2023 17:22:07 -0800 Subject: [PATCH 48/65] Not bytes to file Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 9cb14f90d..78a019eb1 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -275,7 +275,7 @@ def _hashable_repr(self) -> str: indent=None, separators=(',', ': '), sort_keys=False - ).encode('utf-8') + ) def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: """Return list of written files.""" @@ -316,7 +316,7 @@ def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: def _calculate_hash_tree(self) -> dict: prefix = f'RIHS{RIHS_VERSION}_' sha = hashlib.sha256() - sha.update(self._hashable_repr()) + sha.update(self._hashable_repr().encode('utf-8')) type_hash = prefix + sha.hexdigest() type_hash_infos = { From d7442446bf9e5190f40dfcd016ea67ca91269bda Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 28 Feb 2023 15:59:53 -0800 Subject: [PATCH 49/65] Move type hash declarer to rosidl_generator_c Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 5 ++-- rosidl_generator_c/resource/msg__struct.h.em | 3 ++- .../rosidl_generator_c/__init__.py | 25 +++++++++++++++++++ .../resource/action__struct.hpp.em | 3 ++- .../resource/msg__struct.hpp.em | 3 ++- .../resource/srv__struct.hpp.em | 5 +++- .../rosidl_generator_type_hash/__init__.py | 2 +- rosidl_pycommon/rosidl_pycommon/__init__.py | 24 ------------------ 8 files changed, 39 insertions(+), 31 deletions(-) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index 37fa91efb..c9b9d46ab 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -12,6 +12,7 @@ @####################################################################### @{ from rosidl_generator_c import idl_structure_type_to_c_typename +from rosidl_generator_c import type_hash_to_c_definition from rosidl_pycommon import convert_camel_case_to_lower_case_underscore include_parts = [package_name] + list(interface_path.parents[0].parts) + [ 'detail', convert_camel_case_to_lower_case_underscore(interface_path.stem)] @@ -58,7 +59,7 @@ TEMPLATE( from rosidl_parser.definition import Service }@ @[for service in content.get_elements_of_type(Service)]@ -static const @(TYPE_HASH(idl_structure_type_to_c_typename(service.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['service']))@ +static const @(type_hash_to_c_definition(idl_structure_type_to_c_typename(service.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['service']))@ @{ TEMPLATE( @@ -93,7 +94,7 @@ TEMPLATE( from rosidl_parser.definition import Action }@ @[for action in content.get_elements_of_type(Action)]@ -static const @(TYPE_HASH(idl_structure_type_to_c_typename(action.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['action']))@ +static const @(type_hash_to_c_definition(idl_structure_type_to_c_typename(action.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['action']))@ @{ TEMPLATE( diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index f1d7bfee5..c168c2c99 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -21,6 +21,7 @@ from rosidl_generator_c import idl_structure_type_sequence_to_c_typename from rosidl_generator_c import idl_structure_type_to_c_include_prefix from rosidl_generator_c import idl_structure_type_to_c_typename from rosidl_generator_c import interface_path_to_string +from rosidl_generator_c import type_hash_to_c_definition from rosidl_generator_c import value_to_c }@ @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @@ -179,7 +180,7 @@ typedef struct @(idl_structure_type_to_c_typename(message.structure.namespaced_t } @(idl_structure_type_to_c_typename(message.structure.namespaced_type)); // Type Version Hash for interface -static const @(TYPE_HASH(idl_structure_type_to_c_typename(message.structure.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['message']))@ +static const @(type_hash_to_c_definition(idl_structure_type_to_c_typename(message.structure.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['message']))@ @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/rosidl_generator_c/rosidl_generator_c/__init__.py b/rosidl_generator_c/rosidl_generator_c/__init__.py index 17071b67f..a1454c1d0 100644 --- a/rosidl_generator_c/rosidl_generator_c/__init__.py +++ b/rosidl_generator_c/rosidl_generator_c/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + from rosidl_parser.definition import AbstractGenericString from rosidl_parser.definition import AbstractSequence from rosidl_parser.definition import AbstractString @@ -219,3 +221,26 @@ def escape_string(s): def escape_wstring(s): return escape_string(s) + + +def type_hash_to_c_definition(variable_name, hash_string, indent=0): + """Generate empy for rosidl_type_hash_t instance with 8 bytes per line for readability.""" + hash_length = 32 + bytes_per_line = 8 + + indent_str = ' ' * (indent + 2) + pattern = re.compile(r'RIHS([0-9a-f]{2})_([0-9a-f]{64})') + match = pattern.match(hash_string) + if not match: + raise Exception('Type hash string does not match expected RIHS format') + version, value = match.group(1, 2) + + result = f'rosidl_type_hash_t {variable_name} = {{{version}, {{' + for i in range(hash_length): + if i % bytes_per_line == 0: + result += f'\n{indent_str} ' + result += f'0x{value[i * 2:i * 2 + 2]},' + if i % bytes_per_line != bytes_per_line - 1: + result += ' ' + result += f'\n{indent_str}}}}};\n' + return result diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index f34c54014..31dd44649 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -1,5 +1,6 @@ @# Included from rosidl_generator_cpp/resource/idl__struct.hpp.em @{ +from rosidl_generator_c import type_hash_to_c_definition from rosidl_parser.definition import ACTION_FEEDBACK_MESSAGE_SUFFIX from rosidl_parser.definition import ACTION_FEEDBACK_SUFFIX from rosidl_parser.definition import ACTION_GOAL_SERVICE_SUFFIX @@ -78,7 +79,7 @@ namespace @(ns) @[end for]@ struct @(action.namespaced_type.name) { - static constexpr const @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['action'], indent=2))@ + static constexpr const @(type_hash_to_c_definition("TYPE_VERSION_HASH", type_hash['action'], indent=2))@ /// The goal message defined in the action definition. using Goal = @(action_name)@(ACTION_GOAL_SUFFIX); diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index 1bf41e61f..c1c144436 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -1,5 +1,6 @@ @# Included from rosidl_generator_cpp/resource/idl__struct.hpp.em @{ +from rosidl_generator_c import type_hash_to_c_definition from rosidl_generator_cpp import create_init_alloc_and_member_lists from rosidl_generator_cpp import escape_string from rosidl_generator_cpp import escape_wstring @@ -102,7 +103,7 @@ struct @(message.structure.namespaced_type.name)_ using Type = @(message.structure.namespaced_type.name)_; // Type Version Hash for interface - constexpr static const @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['message'], indent=2))@ + constexpr static const @(type_hash_to_c_definition("TYPE_VERSION_HASH", type_hash['message'], indent=2))@ @{ # The creation of the constructors for messages is a bit complicated. The goal diff --git a/rosidl_generator_cpp/resource/srv__struct.hpp.em b/rosidl_generator_cpp/resource/srv__struct.hpp.em index 93a8222bb..8a9f6c4b2 100644 --- a/rosidl_generator_cpp/resource/srv__struct.hpp.em +++ b/rosidl_generator_cpp/resource/srv__struct.hpp.em @@ -1,5 +1,8 @@ @# Included from rosidl_generator_cpp/resource/idl__struct.hpp.em @{ +from rosidl_generator_c import type_hash_to_c_definition +}@ +@{ TEMPLATE( 'msg__struct.hpp.em', package_name=package_name, interface_path=interface_path, @@ -34,7 +37,7 @@ struct @(service.namespaced_type.name) @{ service_typename = '::'.join(service.namespaced_type.namespaced_name()) }@ - static constexpr const @(TYPE_HASH("TYPE_VERSION_HASH", type_hash['service'], indent=2))@ + static constexpr const @(type_hash_to_c_definition("TYPE_VERSION_HASH", type_hash['service'], indent=2))@ using Request = @(service_typename)_Request; using Response = @(service_typename)_Response; diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 78a019eb1..5414f908b 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -22,7 +22,7 @@ from rosidl_parser.parser import parse_idl_file # ROS Interface Hashing Standard, per REP-2011 -RIHS_VERSION = 1 +RIHS_VERSION = '01' def generate_type_hash(generator_arguments_file: str) -> List[str]: diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 419907352..07b1dabb7 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -201,7 +201,6 @@ def expand_template( def _add_helper_functions(data): data['TEMPLATE'] = _expand_template - data['TYPE_HASH'] = _expand_type_hash def _expand_template(template_name, **kwargs): @@ -219,26 +218,3 @@ def _expand_template(template_name, **kwargs): file=sys.stderr) raise interpreter.invoke('afterInclude') - - -def _expand_type_hash(variable_name, hash_string, indent=0): - """Generate empy for rosidl_type_hash_t instance with 8 bytes per line for readability.""" - hash_length = 32 - bytes_per_line = 8 - - indent_str = ' ' * (indent + 2) - pattern = re.compile(r'RIHS(\d+)_([0-9a-f]{64})') - match = pattern.match(hash_string) - if not match: - raise Exception('Type hash string does not match expected RIHS format') - version, value = match.group(1, 2) - - result = f'rosidl_type_hash_t {variable_name} = {{{version}, {{' - for i in range(hash_length): - if i % bytes_per_line == 0: - result += f'\n{indent_str} ' - result += f'0x{value[i * 2:i * 2 + 2]},' - if i % bytes_per_line != bytes_per_line - 1: - result += ' ' - result += f'\n{indent_str}}}}};\n' - return result From 41b1d1379ab284a7ad47e80329063eaebfa27460 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 1 Mar 2023 15:19:14 -0800 Subject: [PATCH 50/65] Reimplement type hash string parsing without sscanf Signed-off-by: Emerson Knapp --- rosidl_runtime_c/src/type_hash.c | 53 ++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index 276cfa43b..0f0fd263b 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -45,9 +45,9 @@ rosidl_stringify_type_hash( return RCUTILS_RET_INVALID_ARGUMENT; } - // Hash representation will be simple hex string, two characters per byte - const char * fmt = "RIHS%d_%64d"; - const size_t prefix_len = strlen("RIHS1_"); + // Hash representation is hex string, two characters per byte + const char * fmt = "RIHS%02d_%64d"; + const size_t prefix_len = strlen("RIHS01_"); char * local_output = rcutils_format_string(allocator, fmt, type_hash->version, 0); if (!local_output) { *output_string = NULL; @@ -62,6 +62,25 @@ rosidl_stringify_type_hash( return RCUTILS_RET_OK; } +static int _xatoi(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'A' && c <= 'F') { + return c - 'A'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a'; + } + return -1; +} + +static int _str_to_byte(const char * str) +{ + return (_xatoi(str[0]) << 4) + _xatoi(str[1]); +} + rcutils_ret_t rosidl_parse_type_hash_string( const char * type_hash_string, @@ -69,25 +88,27 @@ rosidl_parse_type_hash_string( { RCUTILS_CHECK_ARGUMENT_FOR_NULL(type_hash_string, RCUTILS_RET_INVALID_ARGUMENT); RCUTILS_CHECK_ARGUMENT_FOR_NULL(hash_out, RCUTILS_RET_INVALID_ARGUMENT); - static const size_t kValueStringSize = 65; // 32 bytes * 2 digit characters + null-terminator - char hash_value_str[kValueStringSize]; - hash_value_str[kValueStringSize - 1] = '\0'; - int res = sscanf(type_hash_string, "RIHS%hhu_%64s", &hash_out->version, hash_value_str); - if (res != 2) { - RCUTILS_SET_ERROR_MSG("Type hash data did not match expected format."); + static const size_t kprefix_len = sizeof("RIHS01_") - 1; + static const size_t kvalue_len = 64; + hash_out->version = 0; + + if (strlen(type_hash_string) != (kprefix_len + kvalue_len)) { + RCUTILS_SET_ERROR_MSG("Hash string incorrect size."); return RCUTILS_RET_INVALID_ARGUMENT; } - size_t version_digits = log10(hash_out->version) + 1; - size_t prefix_fixed_len = strlen("RIHS_"); - if (strlen(type_hash_string) > kValueStringSize - 1 + prefix_fixed_len + version_digits) { - RCUTILS_SET_ERROR_MSG("Hash value too long."); + if (0 != strncmp(type_hash_string, "RIHS01_", 7)) { + RCUTILS_SET_ERROR_MSG("Type hash string is not prefixed RIHS01_"); return RCUTILS_RET_INVALID_ARGUMENT; } - for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i += 1) { - if (sscanf(hash_value_str + (i * 2), "%2hhx", &hash_out->value[i]) != 1) { - RCUTILS_SET_ERROR_MSG("Couldn't parse hex string of type hash value."); + hash_out->version = 1; + const char * value_str = type_hash_string + kprefix_len; + for (size_t i = 0; i < 32; i++) { + int byte_val = _str_to_byte(value_str + (i * 2)); + if (byte_val < 0) { + RCUTILS_SET_ERROR_MSG("Type hash string value did not contain only hex digits."); return RCUTILS_RET_INVALID_ARGUMENT; } + hash_out->value[i] = (char)byte_val; } return RCUTILS_RET_OK; } From 4843fb1546d8a5f5eecb03f1a09516c048228684 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 1 Mar 2023 15:25:36 -0800 Subject: [PATCH 51/65] Easier to read and maintain, slightly longer, fieldtype serialization Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_hash/__init__.py | 263 +++++++++++++----- 1 file changed, 201 insertions(+), 62 deletions(-) diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 5414f908b..a9f6ee845 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -16,7 +16,7 @@ import json from pathlib import Path import sys -from typing import List +from typing import List, Tuple from rosidl_parser import definition from rosidl_parser.parser import parse_idl_file @@ -80,84 +80,223 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: # This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg # NOTE: Nonexplicit integer types are not defined in FieldType (short, long, long long). # If a ROS IDL uses these, this generator will throw a KeyError. -FIELD_TYPES = { - # 0 reserved for "Not set" - 'nested_type': 1, - 'int8': 2, - 'uint8': 3, - 'int16': 4, - 'uint16': 5, - 'int32': 6, - 'uint32': 7, - 'int64': 8, - 'uint64': 9, - 'float': 10, - 'double': 11, - 'long double': 12, - 'char': 13, - 'wchar': 14, - 'boolean': 15, - 'octet': 16, # byte - definition.UnboundedString: 17, - definition.UnboundedWString: 18, +FIELD_VALUE_TYPE_NAMES = { + None: 'FIELD_TYPE_NOT_SET', + 'nested_type': 'FIELD_TYPE_NESTED_TYPE', + 'int8': 'FIELD_TYPE_INT8', + 'uint8': 'FIELD_TYPE_UINT8', + 'int16': 'FIELD_TYPE_INT16', + 'uint16': 'FIELD_TYPE_UINT16', + 'int32': 'FIELD_TYPE_INT32', + 'uint32': 'FIELD_TYPE_UINT32', + 'int64': 'FIELD_TYPE_INT64', + 'uint64': 'FIELD_TYPE_UINT64', + 'float': 'FIELD_TYPE_FLOAT', + 'double': 'FIELD_TYPE_DOUBLE', + 'long': 'LONG_DOUBLE', + 'char': 'FIELD_TYPE_CHAR', + 'wchar': 'FIELD_TYPE_WCHAR', + 'boolean': 'FIELD_TYPE_BOOLEAN', + 'octet': 'FIELD_TYPE_BYTE', + definition.UnboundedString: 'FIELD_TYPE_STRING', + definition.UnboundedWString: 'FIELD_TYPE_WSTRING', # NOTE: rosidl_parser does not define fixed string types - # FIXED_STRING: 19 - # FIXED_WSTRING: 20 - definition.BoundedString: 21, - definition.BoundedWString: 22, + definition.BoundedString: 'FIELD_TYPE_BOUNDED_STRING', + definition.BoundedWString: 'FIELD_TYPE_BOUNDED_WSTRING', } -FIELD_TYPE_BLOCK_SIZE = 48 -NESTED_FIELD_TYPE_OFFSETS = { - definition.Array: FIELD_TYPE_BLOCK_SIZE, - definition.BoundedSequence: FIELD_TYPE_BLOCK_SIZE * 2, - definition.UnboundedSequence: FIELD_TYPE_BLOCK_SIZE * 3, +NESTED_FIELD_TYPE_SUFFIXES = { + definition.Array: '_ARRAY', + definition.BoundedSequence: '_BOUNDED_SEQUENCE', + definition.UnboundedSequence: '_UNBOUNDED_SEQUENCE', } +# Copied directly from FieldType.msg, with a string replace-all applied +FIELD_TYPE_IDS = { + 'FIELD_TYPE_NOT_SET': 0, + + # Nested type defined in other .msg/.idl files. + 'FIELD_TYPE_NESTED_TYPE': 1, + + # Basic Types + # Integer Types + 'FIELD_TYPE_INT8': 2, + 'FIELD_TYPE_UINT8': 3, + 'FIELD_TYPE_INT16': 4, + 'FIELD_TYPE_UINT16': 5, + 'FIELD_TYPE_INT32': 6, + 'FIELD_TYPE_UINT32': 7, + 'FIELD_TYPE_INT64': 8, + 'FIELD_TYPE_UINT64': 9, + + # Floating-Point Types + 'FIELD_TYPE_FLOAT': 10, + 'FIELD_TYPE_DOUBLE': 11, + 'FIELD_TYPE_LONG_DOUBLE': 12, + + # Char and WChar Types + 'FIELD_TYPE_CHAR': 13, + 'FIELD_TYPE_WCHAR': 14, + + # Boolean Type + 'FIELD_TYPE_BOOLEAN': 15, + + # Byte/Octet Type + 'FIELD_TYPE_BYTE': 16, + + # String Types + 'FIELD_TYPE_STRING': 17, + 'FIELD_TYPE_WSTRING': 18, + + # Fixed String Types + 'FIELD_TYPE_FIXED_STRING': 19, + 'FIELD_TYPE_FIXED_WSTRING': 20, + + # Bounded String Types + 'FIELD_TYPE_BOUNDED_STRING': 21, + 'FIELD_TYPE_BOUNDED_WSTRING': 22, + + # Fixed Sized Array Types + 'FIELD_TYPE_NESTED_TYPE_ARRAY': 49, + 'FIELD_TYPE_INT8_ARRAY': 50, + 'FIELD_TYPE_UINT8_ARRAY': 51, + 'FIELD_TYPE_INT16_ARRAY': 52, + 'FIELD_TYPE_UINT16_ARRAY': 53, + 'FIELD_TYPE_INT32_ARRAY': 54, + 'FIELD_TYPE_UINT32_ARRAY': 55, + 'FIELD_TYPE_INT64_ARRAY': 56, + 'FIELD_TYPE_UINT64_ARRAY': 57, + 'FIELD_TYPE_FLOAT_ARRAY': 58, + 'FIELD_TYPE_DOUBLE_ARRAY': 59, + 'FIELD_TYPE_LONG_DOUBLE_ARRAY': 60, + 'FIELD_TYPE_CHAR_ARRAY': 61, + 'FIELD_TYPE_WCHAR_ARRAY': 62, + 'FIELD_TYPE_BOOLEAN_ARRAY': 63, + 'FIELD_TYPE_BYTE_ARRAY': 64, + 'FIELD_TYPE_STRING_ARRAY': 65, + 'FIELD_TYPE_WSTRING_ARRAY': 66, + 'FIELD_TYPE_FIXED_STRING_ARRAY': 67, + 'FIELD_TYPE_FIXED_WSTRING_ARRAY': 68, + 'FIELD_TYPE_BOUNDED_STRING_ARRAY': 69, + 'FIELD_TYPE_BOUNDED_WSTRING_ARRAY': 70, + + # Bounded Sequence Types + 'FIELD_TYPE_NESTED_TYPE_BOUNDED_SEQUENCE': 97, + 'FIELD_TYPE_INT8_BOUNDED_SEQUENCE': 98, + 'FIELD_TYPE_UINT8_BOUNDED_SEQUENCE': 99, + 'FIELD_TYPE_INT16_BOUNDED_SEQUENCE': 100, + 'FIELD_TYPE_UINT16_BOUNDED_SEQUENCE': 101, + 'FIELD_TYPE_INT32_BOUNDED_SEQUENCE': 102, + 'FIELD_TYPE_UINT32_BOUNDED_SEQUENCE': 103, + 'FIELD_TYPE_INT64_BOUNDED_SEQUENCE': 104, + 'FIELD_TYPE_UINT64_BOUNDED_SEQUENCE': 105, + 'FIELD_TYPE_FLOAT_BOUNDED_SEQUENCE': 106, + 'FIELD_TYPE_DOUBLE_BOUNDED_SEQUENCE': 107, + 'FIELD_TYPE_LONG_DOUBLE_BOUNDED_SEQUENCE': 108, + 'FIELD_TYPE_CHAR_BOUNDED_SEQUENCE': 109, + 'FIELD_TYPE_WCHAR_BOUNDED_SEQUENCE': 110, + 'FIELD_TYPE_BOOLEAN_BOUNDED_SEQUENCE': 111, + 'FIELD_TYPE_BYTE_BOUNDED_SEQUENCE': 112, + 'FIELD_TYPE_STRING_BOUNDED_SEQUENCE': 113, + 'FIELD_TYPE_WSTRING_BOUNDED_SEQUENCE': 114, + 'FIELD_TYPE_FIXED_STRING_BOUNDED_SEQUENCE': 115, + 'FIELD_TYPE_FIXED_WSTRING_BOUNDED_SEQUENCE': 116, + 'FIELD_TYPE_BOUNDED_STRING_BOUNDED_SEQUENCE': 117, + 'FIELD_TYPE_BOUNDED_WSTRING_BOUNDED_SEQUENCE': 118, + + # Unbounded Sequence Types + 'FIELD_TYPE_NESTED_TYPE_UNBOUNDED_SEQUENCE': 145, + 'FIELD_TYPE_INT8_UNBOUNDED_SEQUENCE': 146, + 'FIELD_TYPE_UINT8_UNBOUNDED_SEQUENCE': 147, + 'FIELD_TYPE_INT16_UNBOUNDED_SEQUENCE': 148, + 'FIELD_TYPE_UINT16_UNBOUNDED_SEQUENCE': 149, + 'FIELD_TYPE_INT32_UNBOUNDED_SEQUENCE': 150, + 'FIELD_TYPE_UINT32_UNBOUNDED_SEQUENCE': 151, + 'FIELD_TYPE_INT64_UNBOUNDED_SEQUENCE': 152, + 'FIELD_TYPE_UINT64_UNBOUNDED_SEQUENCE': 153, + 'FIELD_TYPE_FLOAT_UNBOUNDED_SEQUENCE': 154, + 'FIELD_TYPE_DOUBLE_UNBOUNDED_SEQUENCE': 155, + 'FIELD_TYPE_LONG_DOUBLE_UNBOUNDED_SEQUENCE': 156, + 'FIELD_TYPE_CHAR_UNBOUNDED_SEQUENCE': 157, + 'FIELD_TYPE_WCHAR_UNBOUNDED_SEQUENCE': 158, + 'FIELD_TYPE_BOOLEAN_UNBOUNDED_SEQUENCE': 159, + 'FIELD_TYPE_BYTE_UNBOUNDED_SEQUENCE': 160, + 'FIELD_TYPE_STRING_UNBOUNDED_SEQUENCE': 161, + 'FIELD_TYPE_WSTRING_UNBOUNDED_SEQUENCE': 162, + 'FIELD_TYPE_FIXED_STRING_UNBOUNDED_SEQUENCE': 163, + 'FIELD_TYPE_FIXED_WSTRING_UNBOUNDED_SEQUENCE': 164, + 'FIELD_TYPE_BOUNDED_STRING_UNBOUNDED_SEQUENCE': 165, + 'FIELD_TYPE_BOUNDED_WSTRING_UNBOUNDED_SEQUENCE': 166, +} -def serialize_field_type(ftype: definition.AbstractType) -> dict: - result = { - 'type_id': 0, - 'length': 0, - 'string_length': 0, - 'nested_type_name': '', - } - # Determine value type, if this is a nested type - type_id_offset = 0 - if isinstance(ftype, definition.AbstractNestableType): - value_type = ftype - elif isinstance(ftype, definition.AbstractNestedType): - type_id_offset = NESTED_FIELD_TYPE_OFFSETS[type(ftype)] +def field_type_type_name(ftype: definition.AbstractType) -> str: + value_type = ftype + name_suffix = '' + + if isinstance(ftype, definition.AbstractNestedType): value_type = ftype.value_type + name_suffix = NESTED_FIELD_TYPE_SUFFIXES[type(ftype)] + + if isinstance(value_type, definition.BasicType): + value_type_name = FIELD_VALUE_TYPE_NAMES[value_type.typename] + elif isinstance(value_type, definition.AbstractGenericString): + value_type_name = FIELD_VALUE_TYPE_NAMES[type(value_type)] + elif ( + isinstance(value_type, definition.NamespacedType) or + isinstance(value_type, definition.NamedType) + ): + value_type_name = 'FIELD_TYPE_NESTED_TYPE' + + return value_type_name + name_suffix + + +def field_type_type_id(ftype: definition.AbstractType) -> Tuple[str, int]: + return FIELD_TYPE_IDS[field_type_type_name(ftype)] + + +def field_type_length(ftype: definition.AbstractType): + if isinstance(ftype, definition.AbstractNestedType): if ftype.has_maximum_size(): try: - result['length'] = ftype.maximum_size + return ftype.maximum_size except AttributeError: - result['length'] = ftype.size - else: - raise Exception('Unable to translate field type', ftype) + return ftype.size + return 0 - # Translate value type to FieldType.msg const value - if isinstance(value_type, definition.BasicType): - result['type_id'] = FIELD_TYPES[value_type.typename] + type_id_offset - elif isinstance(value_type, definition.AbstractGenericString): - result['type_id'] = FIELD_TYPES[type(value_type)] + type_id_offset + +def field_type_string_length(ftype: definition.AbstractType): + value_type = ftype + if isinstance(ftype, definition.AbstractNestedType): + value_type = ftype.value_type + + if isinstance(value_type, definition.AbstractGenericString): if value_type.has_maximum_size(): try: - result['string_length'] = value_type.maximum_size + return value_type.maximum_size except AttributeError: - result['string_length'] = value_type.size - elif isinstance(value_type, definition.NamespacedType): - result['type_id'] = type_id_offset - result['nested_type_name'] = '/'.join(value_type.namespaced_name()) + return value_type.size + return 0 + + +def field_type_nested_type_name(ftype: definition.AbstractType, joiner='/'): + value_type = ftype + if isinstance(ftype, definition.AbstractNestedType): + value_type = ftype.value_type + if isinstance(value_type, definition.NamespacedType): + return joiner.join(value_type.namespaced_name()) elif isinstance(value_type, definition.NamedType): - result['type_id'] = type_id_offset - result['nested_type_name'] = value_type.name - else: - raise TypeError('Unknown value type ', value_type) + return value_type.name + return '' + - return result +def serialize_field_type(ftype: definition.AbstractType) -> dict: + return { + 'type_id': field_type_type_id(ftype), + 'length': field_type_length(ftype), + 'string_length': field_type_string_length(ftype), + 'nested_type_name': field_type_nested_type_name(ftype), + } def serialize_field(member: definition.Member) -> dict: From f8032910617d86f60d2792e6dde29e82c7fcc9c7 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Wed, 1 Mar 2023 16:05:22 -0800 Subject: [PATCH 52/65] Use new FieldType field names Signed-off-by: Emerson Knapp --- .../resource/IndividualTypeDescription.schema.json | 6 +++--- .../rosidl_generator_type_hash/__init__.py | 8 ++++---- .../test/test_serializers.py | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json b/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json index 5e82291fd..6f3a16111 100644 --- a/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json +++ b/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json @@ -29,11 +29,11 @@ "type": "object", "properties": { "type_id": {"type": "integer", "minimum": 0, "maximum": 255}, - "length": {"type": "integer", "minimum": 0}, - "string_length": {"type": "integer", "minimum": 0}, + "capacity": {"type": "integer", "minimum": 0}, + "string_capacity": {"type": "integer", "minimum": 0}, "nested_type_name": {"type": "string", "maxLength": 255} }, - "required": ["type_id", "length", "string_length", "nested_type_name"], + "required": ["type_id", "capacity", "string_capacity", "nested_type_name"], "additionalProperties": false } } diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index a9f6ee845..57d0a5926 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -255,7 +255,7 @@ def field_type_type_id(ftype: definition.AbstractType) -> Tuple[str, int]: return FIELD_TYPE_IDS[field_type_type_name(ftype)] -def field_type_length(ftype: definition.AbstractType): +def field_type_capacity(ftype: definition.AbstractType): if isinstance(ftype, definition.AbstractNestedType): if ftype.has_maximum_size(): try: @@ -265,7 +265,7 @@ def field_type_length(ftype: definition.AbstractType): return 0 -def field_type_string_length(ftype: definition.AbstractType): +def field_type_string_capacity(ftype: definition.AbstractType): value_type = ftype if isinstance(ftype, definition.AbstractNestedType): value_type = ftype.value_type @@ -293,8 +293,8 @@ def field_type_nested_type_name(ftype: definition.AbstractType, joiner='/'): def serialize_field_type(ftype: definition.AbstractType) -> dict: return { 'type_id': field_type_type_id(ftype), - 'length': field_type_length(ftype), - 'string_length': field_type_string_length(ftype), + 'capacity': field_type_capacity(ftype), + 'string_capacity': field_type_string_capacity(ftype), 'nested_type_name': field_type_nested_type_name(ftype), } diff --git a/rosidl_generator_type_hash/test/test_serializers.py b/rosidl_generator_type_hash/test/test_serializers.py index c52bd930a..74654cafa 100644 --- a/rosidl_generator_type_hash/test/test_serializers.py +++ b/rosidl_generator_type_hash/test/test_serializers.py @@ -18,14 +18,14 @@ def test_field_type_serializer(): - # Sanity check for the more complex length/string_length types and nesting + # Sanity check for the more complex capacity/string_capacity types and nesting string_limit = 12 array_size = 22 test_type = definition.Array(definition.BoundedString(string_limit), array_size) expected = { 'type_id': 69, - 'length': array_size, - 'string_length': string_limit, + 'capacity': array_size, + 'string_capacity': string_limit, 'nested_type_name': '', } @@ -36,8 +36,8 @@ def test_field_type_serializer(): test_type = definition.BoundedSequence(definition.UnboundedString(), bounded_sequence_limit) expected = { 'type_id': 113, - 'length': bounded_sequence_limit, - 'string_length': 0, + 'capacity': bounded_sequence_limit, + 'string_capacity': 0, 'nested_type_name': '', } result = serialize_field_type(test_type) @@ -46,8 +46,8 @@ def test_field_type_serializer(): test_type = definition.BoundedWString(string_limit) expected = { 'type_id': 22, - 'length': 0, - 'string_length': string_limit, + 'capacity': 0, + 'string_capacity': string_limit, 'nested_type_name': '', } result = serialize_field_type(test_type) From b22b6c69b935095b3f7621c27b9f466790ad81f3 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 2 Mar 2023 14:33:43 -0800 Subject: [PATCH 53/65] Type hashes single-file simplified flow Signed-off-by: Emerson Knapp --- .../test_type_hash.py | 44 ++--- rosidl_generator_type_hash/README.md | 33 +--- ...erator_type_hash_generate_interfaces.cmake | 27 +--- .../HashedTypeDescription.schema.json | 114 +++++++++++++ .../IndividualTypeDescription.schema.json | 40 ----- .../resource/TypeDescription.schema.json | 18 --- .../resource/TypeDescriptionIn.schema.json | 16 -- .../resource/TypeVersionHash.schema.json | 60 ------- .../rosidl_generator_type_hash/__init__.py | 153 ++++++++---------- rosidl_pycommon/rosidl_pycommon/__init__.py | 2 +- rosidl_runtime_c/src/type_hash.c | 4 +- rosidl_runtime_c/test/test_type_hash.cpp | 18 +-- 12 files changed, 217 insertions(+), 312 deletions(-) create mode 100644 rosidl_generator_type_hash/resource/HashedTypeDescription.schema.json delete mode 100644 rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json delete mode 100644 rosidl_generator_type_hash/resource/TypeDescription.schema.json delete mode 100644 rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json delete mode 100644 rosidl_generator_type_hash/resource/TypeVersionHash.schema.json diff --git a/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py b/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py index 653e5b0b0..4adf49013 100644 --- a/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py +++ b/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py @@ -22,46 +22,20 @@ def test_type_hash(): """Test all rosidl_generator_type_hash output files against defined schemas.""" - schema_dir = Path(get_package_share_directory('rosidl_generator_type_hash')) / 'resource' - resolver = jsonschema.validators.RefResolver( - base_uri=f'{schema_dir.as_uri()}/', - referrer=True, - ) + schema_path = ( + Path(get_package_share_directory('rosidl_generator_type_hash')) / 'resource' / + 'HashedTypeDescription.schema.json') + with schema_path.open('r') as schema_file: + schema = json.load(schema_file) generated_files_dir = Path(os.environ['GENERATED_TEST_FILE_DIR']) - validated_sha256 = 0 - validated_json_in = 0 - validated_json = 0 + validated_files = 0 for namespace in generated_files_dir.iterdir(): for p in namespace.iterdir(): assert p.is_file() assert p.suffix == '.json' with p.open('r') as f: instance = json.load(f) - subsuffix = p.with_suffix('').suffix - if subsuffix == '.sha256': - jsonschema.validate( - instance=instance, - schema={'$ref': 'TypeVersionHash.schema.json'}, - resolver=resolver, - ) - validated_sha256 += 1 - elif subsuffix == '.in': - jsonschema.validate( - instance=instance, - schema={'$ref': 'TypeDescriptionIn.schema.json'}, - resolver=resolver, - ) - validated_json_in += 1 - elif subsuffix == '': - jsonschema.validate( - instance=instance, - schema={'$ref': 'TypeDescription.schema.json'}, - resolver=resolver, - ) - validated_json += 1 - else: - assert False, 'Unknown file type to validate' - assert validated_sha256, 'Needed to validate at least one of each type of file.' - assert validated_json_in, 'Needed to validate at least one of each type of file.' - assert validated_json, 'Needed to validate at least one of each type of file.' + jsonschema.validate(instance=instance, schema=schema) + validated_files += 1 + assert validated_files, 'Needed to validate at least one JSON output.' diff --git a/rosidl_generator_type_hash/README.md b/rosidl_generator_type_hash/README.md index 4279b4b9b..3d1bdb950 100644 --- a/rosidl_generator_type_hash/README.md +++ b/rosidl_generator_type_hash/README.md @@ -6,32 +6,9 @@ The SHA256 hashes generated by this package must match those generated by `rcl_c ## Generated files -This generator creates the following output files from parsed IDL interface descriptions: +This generator creates one output file per interface, `interface_name.json`. -* `interface_type.sha256.json` -* `interface_type.in.json` -* `interface_type.json` - -See the schema files for an exact description for the contents of the generated files. - -### `.sha256.json` - -**Schema**: [TypeVersionHash](./resource/TypeVersionHash.schema.json) - -The true output of the generator, containing hashes for top-level interface and sub-interfaces if any are present, to be consumed by code generators. - -The remaining generated files are intermediate artifacts that lead to the output hashes. - -### `.json` - -**Schema**: [TypeDescription](./resource/TypeDescription.schema.json) - -The fully-expanded serialized equivalent of `type_description_interfaces/msg/TypeDescription`, whose contents are hashed. - -NOTE: `.json` contents must have all whitespace stripped before hashing. This package generates json without whitespace, including having no trailing newline. - -### `.in.json` - -**Schema**: [TypeDescriptionIn](./resource/TypeDescriptionIn.schema.json) - -The `IndividualTypeDescription`-equivalent serialization, and a list of referenced `.in.json` files to be used when expanding to `.json`. This file is used by dependent interfaces, especially in dependent packages, when expanding to `.json` so IDL files need only be parsed a single time when running the generator on their own package. +This file follows the schema [`HashedTypedDescription`](./resource/HashedTypeDescription.schema.json). +It contains a tree of hashes for the top-level interface and any of its generated subinterfaces (such as request and response messages for a service), as well as fully-expanded descriptions of the interface type. +This description is a representation of `type_description_interfaces/msg/TypeDescription`, including all recursively-referenced types. +This way, dependent descriptions may use this interface and recurse no further to know the full set of referenced types it needs to know about. diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake index fa54f66fc..d9bd2a24b 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake @@ -15,40 +15,30 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash/${PROJECT_NAME}") -set(_generated_json_in "") -set(_generated_json "") -set(_generated_hash_tuples "") -set(_generated_hash_files "") +set(_generated_files "") +set(_generated_tuples "") -# Create lists of generated fiiles +# Create list of generated files foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) get_filename_component(_parent_folder "${_abs_idl_file}" DIRECTORY) get_filename_component(_parent_folder "${_parent_folder}" NAME) get_filename_component(_idl_stem "${_abs_idl_file}" NAME_WE) - list(APPEND _generated_json_in - "${_output_path}/${_parent_folder}/${_idl_stem}.in.json") - list(APPEND _generated_json - "${_output_path}/${_parent_folder}/${_idl_stem}.json") - set(_hash_file "${_output_path}/${_parent_folder}/${_idl_stem}.sha256.json") - list(APPEND _generated_hash_files ${_hash_file}) - list(APPEND _generated_hash_tuples "${_parent_folder}/${_idl_stem}:${_hash_file}") + set(_json_file "${_output_path}/${_parent_folder}/${_idl_stem}.json") + list(APPEND _generated_files "${_json_file}") + list(APPEND _generated_tuples "${_parent_folder}/${_idl_stem}:${_json_file}") endforeach() -# Find dependency packages' .in.json files +# Find dependency packages' generated files set(_dependency_files "") set(_dependency_paths "") foreach(_pkg_name ${rosidl_generate_interfaces_DEPENDENCY_PACKAGE_NAMES}) set(_include_path "${${_pkg_name}_DIR}/..") normalize_path(_include_path "${_include_path}") list(APPEND _dependency_paths "${_pkg_name}:${_include_path}") - foreach(_json_in_file ${${_pkg_name}_JSON_IN_FILES}) - set(_abs_json_in_file ${_include_path}/${_json_in_file}) - list(APPEND _dependency_files ${_abs_json_in_file}) - endforeach() endforeach() # Export __HASH_TUPLES variable for use by dependent generators -set(${rosidl_generate_interfaces_TARGET}__HASH_TUPLES ${_generated_hash_tuples}) +set(${rosidl_generate_interfaces_TARGET}__HASH_TUPLES ${_generated_tuples}) # Validate that all dependencies exist set(target_dependencies @@ -72,7 +62,6 @@ rosidl_write_generator_arguments( ) # Create custom command and target to generate the hash output -set(_generated_files "${_generated_json};${_generated_json_in};${_generated_hash_files}") add_custom_command( COMMAND Python3::Interpreter ARGS diff --git a/rosidl_generator_type_hash/resource/HashedTypeDescription.schema.json b/rosidl_generator_type_hash/resource/HashedTypeDescription.schema.json new file mode 100644 index 000000000..e1b9d05db --- /dev/null +++ b/rosidl_generator_type_hash/resource/HashedTypeDescription.schema.json @@ -0,0 +1,114 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "HashedTypeDescription.schema.json", + "title": "HashedTypeDescription", + "description": "TODO", + "type": "object", + "properties": { + "hashes": { + "type": "object", + "oneOf": [ + { "$ref": "#/$defs/MessageHash" }, + { "$ref": "#/$defs/ServiceHashes" }, + { "$ref": "#/$defs/ActionHashes" } + ] + }, + "type_description": { "$ref": "#/$defs/TypeDescription" } + }, + "required": ["hashes", "type_description"], + "additionalProperties": false, + "$defs": { + "MessageHash": { + "type": "object", + "properties": { + "message": { "type": "string" } + }, + "required": [ "message" ], + "additionalProperties": false + }, + "ServiceHashes": { + "type": "object", + "properties": { + "service": { "type": "string" }, + "request_message": { "$ref": "#/$defs/MessageHash" }, + "response_message": { "$ref": "#/$defs/MessageHash" }, + "event_message": { "$ref": "#/$defs/MessageHash" } + }, + "required": [ + "service", + "request_message", + "response_message", + "event_message" + ], + "additionalProperties": false + }, + "ActionHashes": { + "type": "object", + "properties": { + "action": { "type": "string" }, + "goal": { "$ref": "#/$defs/MessageHash" }, + "result": { "$ref": "#/$defs/MessageHash" }, + "feedback": { "$ref": "#/$defs/MessageHash" }, + "send_goal_service": { "$ref": "#/$defs/ServiceHashes" }, + "get_result_service": { "$ref": "#/$defs/ServiceHashes" }, + "feedback_message": { "$ref": "#/$defs/MessageHash" } + }, + "required": [ + "action", + "goal", + "result", + "feedback", + "send_goal_service", + "get_result_service", + "feedback_message" + ], + "additionalProperties": false + }, + "TypeDescription": { + "type": "object", + "$comment": "All whitespace must be excluded for consistent hashing, which this schema cannot enforce.", + "properties": { + "type_description": {"$ref": "#/$defs/IndividualTypeDescription"}, + "referenced_type_descriptions": { + "$comment": "Referenced type descriptions must be alphabetized, which this schema cannot enforce.", + "type": "array", + "items": { "$ref": "#/$defs/IndividualTypeDescription" } + } + }, + "required": ["type_description", "referenced_type_descriptions"], + "additionalProperties": false + }, + "IndividualTypeDescription": { + "type": "object", + "properties": { + "type_name": {"type": "string", "maxLength": 255}, + "fields": { + "type": "array", + "items": { "$ref": "#/$defs/Field" } + } + }, + "required": ["type_name", "fields"], + "additionalProperties": false + }, + "Field": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "type": {"$ref": "#/$defs/FieldType"} + }, + "required": ["name", "type"], + "additionalProperties": false + }, + "FieldType": { + "type": "object", + "properties": { + "type_id": {"type": "integer", "minimum": 0, "maximum": 255}, + "capacity": {"type": "integer", "minimum": 0}, + "string_capacity": {"type": "integer", "minimum": 0}, + "nested_type_name": {"type": "string", "maxLength": 255} + }, + "required": ["type_id", "capacity", "string_capacity", "nested_type_name"], + "additionalProperties": false + } + } +} diff --git a/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json b/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json deleted file mode 100644 index 6f3a16111..000000000 --- a/rosidl_generator_type_hash/resource/IndividualTypeDescription.schema.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "IndividualTypeDescription.schema.json", - "title": "IndividualTypeDescription", - "description": "type_description_interfaces/msg/IndividualTypeDescription.msg JSON representation", - "type": "object", - "properties": { - "type_name": {"type": "string", "maxLength": 255}, - "fields": { - "type": "array", - "items": { - "$ref": "#/$defs/Field" - } - } - }, - "required": ["type_name", "fields"], - "additionalProperties": false, - "$defs": { - "Field": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "type": {"$ref": "#/$defs/FieldType"} - }, - "required": ["name", "type"], - "additionalProperties": false - }, - "FieldType": { - "type": "object", - "properties": { - "type_id": {"type": "integer", "minimum": 0, "maximum": 255}, - "capacity": {"type": "integer", "minimum": 0}, - "string_capacity": {"type": "integer", "minimum": 0}, - "nested_type_name": {"type": "string", "maxLength": 255} - }, - "required": ["type_id", "capacity", "string_capacity", "nested_type_name"], - "additionalProperties": false - } - } -} diff --git a/rosidl_generator_type_hash/resource/TypeDescription.schema.json b/rosidl_generator_type_hash/resource/TypeDescription.schema.json deleted file mode 100644 index c4ec81a40..000000000 --- a/rosidl_generator_type_hash/resource/TypeDescription.schema.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "TypeDescription.schema.json", - "title": "TypeDescription", - "description": "type_description_interfaces/msg/TypeDescription.msg JSON representation", - "type": "object", - "$comment": "All whitespace must be excluded for consistent hashing, which this schema cannot enforce.", - "properties": { - "type_description": {"$ref": "file:IndividualTypeDescription.schema.json"}, - "referenced_type_descriptions": { - "$comment": "Referenced type descriptions must be alphabetized, which this schema cannot enforce.", - "type": "array", - "items": { "$ref": "file:IndividualTypeDescription.schema.json" } - } - }, - "required": ["type_description", "referenced_type_descriptions"], - "additionalProperties": false -} diff --git a/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json b/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json deleted file mode 100644 index 8be941749..000000000 --- a/rosidl_generator_type_hash/resource/TypeDescriptionIn.schema.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "TypeDescriptionIn.schema.json", - "title": "TypeDescriptionIn", - "description": "Input for TypeDescription expansion, providing include paths for references", - "type": "object", - "properties": { - "type_description": {"$ref": "file:IndividualTypeDescription.schema.json"}, - "includes": { - "type": "array", - "items": { "type": "string" } - } - }, - "required": ["type_description", "includes"], - "additionalProperties": false -} diff --git a/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json b/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json deleted file mode 100644 index fe88c3548..000000000 --- a/rosidl_generator_type_hash/resource/TypeVersionHash.schema.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "TypeVersionHash.schema.json", - "title": "TypeVersionHash", - "description": "A tree of hashes (and sub-hashes) for ROS 2 interfaces", - "type": "object", - "$defs": { - "message": { - "type": "object", - "properties": { - "message": { "type": "string" } - }, - "required": [ "message" ], - "additionalProperties": false - }, - "service": { - "type": "object", - "properties": { - "service": { "type": "string" }, - "request_message": { "$ref": "#/$defs/message" }, - "response_message": { "$ref": "#/$defs/message" }, - "event_message": { "$ref": "#/$defs/message" } - }, - "required": [ - "service", - "request_message", - "response_message", - "event_message" - ], - "additionalProperties": false - }, - "action": { - "type": "object", - "properties": { - "action": { "type": "string" }, - "goal": { "$ref": "#/$defs/message" }, - "result": { "$ref": "#/$defs/message" }, - "feedback": { "$ref": "#/$defs/message" }, - "send_goal_service": { "$ref": "#/$defs/service" }, - "get_result_service": { "$ref": "#/$defs/service" }, - "feedback_message": { "$ref": "#/$defs/message" } - }, - "required": [ - "action", - "goal", - "result", - "feedback", - "send_goal_service", - "get_result_service", - "feedback_message" - ], - "additionalProperties": false - } - }, - "oneOf": [ - { "$ref": "#/$defs/message" }, - { "$ref": "#/$defs/service" }, - { "$ref": "#/$defs/action" } - ] -} diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py index 57d0a5926..73aeea0ea 100644 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py +++ b/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py @@ -33,7 +33,7 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: idl_tuples = args['idl_tuples'] include_paths = args.get('include_paths', []) - # Lookup for directory containing dependency .in.json files + # Lookup for directory containing dependency .json files include_map = { package_name: output_dir } @@ -44,9 +44,9 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: include_map[include_package_name] = Path(include_base_path) generated_files = [] - hashers = [] + hashers = {} - # First generate all .in.json files (so referenced types can be used in expansion) + # Initialize all local types first so they can be referenced by other local types for idl_tuple in idl_tuples: idl_parts = idl_tuple.rsplit(':', 1) assert len(idl_parts) == 2 @@ -63,16 +63,11 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: generate_to_dir.mkdir(parents=True, exist_ok=True) hasher = InterfaceHasher.from_idl(idl_file) - generated_files.extend( - hasher.write_json_in(output_dir)) - hashers.append(hasher) + hashers[hasher.namespaced_type.namespaced_name()] = hasher - # Expand .in.json and generate .sha256.json hash files - for hasher in hashers: - generated_files.extend( - hasher.write_json_out(output_dir, include_map)) - generated_files.extend( - hasher.write_hash(output_dir)) + # Generate output files + for hasher in hashers.values(): + generated_files += hasher.write_unified_json(output_dir, hashers, include_map) return generated_files @@ -111,7 +106,7 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: definition.UnboundedSequence: '_UNBOUNDED_SEQUENCE', } -# Copied directly from FieldType.msg, with a string replace-all applied +# Copied directly from FieldType.msg, with simple string manipulation to create a dict FIELD_TYPE_IDS = { 'FIELD_TYPE_NOT_SET': 0, @@ -255,7 +250,7 @@ def field_type_type_id(ftype: definition.AbstractType) -> Tuple[str, int]: return FIELD_TYPE_IDS[field_type_type_name(ftype)] -def field_type_capacity(ftype: definition.AbstractType): +def field_type_capacity(ftype: definition.AbstractType) -> int: if isinstance(ftype, definition.AbstractNestedType): if ftype.has_maximum_size(): try: @@ -265,7 +260,7 @@ def field_type_capacity(ftype: definition.AbstractType): return 0 -def field_type_string_capacity(ftype: definition.AbstractType): +def field_type_string_capacity(ftype: definition.AbstractType) -> int: value_type = ftype if isinstance(ftype, definition.AbstractNestedType): value_type = ftype.value_type @@ -279,7 +274,7 @@ def field_type_string_capacity(ftype: definition.AbstractType): return 0 -def field_type_nested_type_name(ftype: definition.AbstractType, joiner='/'): +def field_type_nested_type_name(ftype: definition.AbstractType, joiner='/') -> str: value_type = ftype if isinstance(ftype, definition.AbstractNestedType): value_type = ftype.value_type @@ -369,93 +364,91 @@ def __init__(self, interface): self.namespaced_type, self.members) # Determine needed includes from member fields - included_types = [] + self.includes = [] for member in self.members: if isinstance(member.type, definition.NamespacedType): - included_types.append(member.type) + self.includes.append(member.type.namespaced_name()) elif ( isinstance(member.type, definition.AbstractNestedType) and isinstance(member.type.value_type, definition.NamespacedType) ): - included_types.append(member.type.value_type) - - self.includes = [ - str(Path(*t.namespaced_name()).with_suffix('.in.json')) - for t in included_types - ] + self.includes.append(member.type.value_type.namespaced_name()) self.rel_path = Path(*self.namespaced_type.namespaced_name()[1:]) self.include_path = Path(*self.namespaced_type.namespaced_name()) - self.json_in = { - 'type_description': self.individual_type_description, - 'includes': self.includes, - } - - def write_json_in(self, output_dir) -> List[str]: - """Return list of written files.""" + def write_unified_json( + self, output_dir: Path, local_hashers: dict, includes_map: dict + ) -> List[Path]: generated_files = [] - for key, val in self.subinterfaces.items(): - generated_files += val.write_json_in(output_dir) - - json_path = output_dir / self.rel_path.with_suffix('.in.json') - json_path.parent.mkdir(parents=True, exist_ok=True) - with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(json.dumps(self.json_in, indent=2)) - return generated_files + [str(json_path)] - - def _hashable_repr(self) -> str: - return json.dumps( - self.json_out, - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=False, - indent=None, - separators=(',', ': '), - sort_keys=False - ) + referenced_types = {} - def write_json_out(self, output_dir: Path, includes_map: dict) -> List[str]: - """Return list of written files.""" - generated_files = [] for key, val in self.subinterfaces.items(): - generated_files += val.write_json_out(output_dir, includes_map) - - # Recursively load includes from all included type descriptions - pending_includes = self.includes[:] - loaded_includes = {} - while pending_includes: - process_include = pending_includes.pop() - if process_include in loaded_includes: + generated_files += val.write_unified_json(output_dir, local_hashers, includes_map) + + def add_referenced_type(individual_type_description): + type_name = individual_type_description['type_name'] + if ( + type_name in referenced_types and + referenced_types[type_name] != individual_type_description + ): + raise Exception('Encountered two definitions of the same referenced type') + referenced_types[type_name] = individual_type_description + + process_includes = self.includes[:] + while process_includes: + process_type = process_includes.pop() + + # A type in this package may refer to types, and hasn't been unrolled yet, + # so process its includes breadth first + if process_type in local_hashers: + add_referenced_type(local_hashers[process_type].individual_type_description) + process_includes += local_hashers[process_type].includes continue - p_path = Path(process_include) - assert(not p_path.is_absolute()) - include_package = p_path.parts[0] - include_file = includes_map[include_package] / p_path.relative_to(include_package) - with include_file.open('r') as include_file: + # All nonlocal descriptions will have all recursively referenced types baked in + p_path = Path(*process_type).with_suffix('.json') + pkg = p_path.parts[0] + pkg_dir = includes_map[pkg] + include_path = pkg_dir / p_path.relative_to(pkg) + with include_path.open('r') as include_file: include_json = json.load(include_file) - loaded_includes[process_include] = include_json['type_description'] - pending_includes.extend(include_json['includes']) + type_description = include_json['type_description'] + add_referenced_type(type_description['type_description']) + for rt in type_description['referenced_type_descriptions']: + add_referenced_type(rt) - # Sort included type descriptions alphabetically by type name - self.json_out = { - 'type_description': self.json_in['type_description'], + self.full_type_description = { + 'type_description': self.individual_type_description, 'referenced_type_descriptions': sorted( - loaded_includes.values(), key=lambda td: td['type_name']) + referenced_types.values(), key=lambda td: td['type_name']) + } + + hashed_type_description = { + 'hashes': self._calculate_hash_tree(), + 'type_description': self.full_type_description, } json_path = output_dir / self.rel_path.with_suffix('.json') with json_path.open('w', encoding='utf-8') as json_file: - json_file.write(self._hashable_repr()) - return generated_files + [str(json_path)] + json_file.write(json.dumps(hashed_type_description, indent=2)) + return generated_files + [json_path] def _calculate_hash_tree(self) -> dict: + hashable_repr = json.dumps( + self.full_type_description, + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=False, + indent=None, + separators=(',', ': '), + sort_keys=False + ) prefix = f'RIHS{RIHS_VERSION}_' sha = hashlib.sha256() - sha.update(self._hashable_repr().encode('utf-8')) + sha.update(hashable_repr.encode('utf-8')) type_hash = prefix + sha.hexdigest() type_hash_infos = { @@ -465,11 +458,3 @@ def _calculate_hash_tree(self) -> dict: type_hash_infos[key] = val._calculate_hash_tree() return type_hash_infos - - def write_hash(self, output_dir: Path) -> List[str]: - """Return list of written files.""" - type_hash = self._calculate_hash_tree() - hash_path = output_dir / self.rel_path.with_suffix('.sha256.json') - with hash_path.open('w', encoding='utf-8') as hash_file: - hash_file.write(json.dumps(type_hash, indent=2)) - return [str(hash_path)] diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 07b1dabb7..536e574ba 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -79,7 +79,7 @@ def generate_files( if type_hashes_provided: type_hash_file = type_hash_files[str(idl_rel_stem)] with open(type_hash_file, 'r') as f: - type_hash_infos = json.load(f) + type_hash_infos = json.load(f)['hashes'] else: type_hash_infos = None diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index 0f0fd263b..46cba9fbf 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -68,10 +68,10 @@ static int _xatoi(char c) return c - '0'; } if (c >= 'A' && c <= 'F') { - return c - 'A'; + return c - 'A' + 0xa; } if (c >= 'a' && c <= 'f') { - return c - 'a'; + return c - 'a' + 0xa; } return -1; } diff --git a/rosidl_runtime_c/test/test_type_hash.cpp b/rosidl_runtime_c/test/test_type_hash.cpp index 895e066d6..8d68a2da2 100644 --- a/rosidl_runtime_c/test/test_type_hash.cpp +++ b/rosidl_runtime_c/test/test_type_hash.cpp @@ -27,7 +27,7 @@ TEST(type_hash, init_zero_hash) { TEST(type_hash, stringify_basic) { const std::string expected = - "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + "RIHS01_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); hash.version = 1; for (size_t i = 0; i < sizeof(hash.value); i++) { @@ -44,7 +44,7 @@ TEST(type_hash, stringify_basic) { TEST(type_hash, parse_basic) { const std::string test_value = - "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + "RIHS01_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); ASSERT_EQ(RCUTILS_RET_OK, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); @@ -57,7 +57,7 @@ TEST(type_hash, parse_basic) { TEST(type_hash, parse_bad_prefix) { const std::string test_value = - "RRRR1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + "RRRR01_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); rcutils_reset_error(); @@ -73,7 +73,7 @@ TEST(type_hash, parse_no_version) { TEST(type_hash, parse_too_short) { const std::string test_value = - "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddee"; + "RIHS01_00112233445566778899aabbccddeeff00112233445566778899aabbccddee"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); rcutils_reset_error(); @@ -81,16 +81,16 @@ TEST(type_hash, parse_too_short) { TEST(type_hash, parse_too_long) { const std::string test_value = - "RIHS1_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff00"; + "RIHS01_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff00"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); rcutils_reset_error(); } -TEST(type_hash, parse_multidigit_version) { +TEST(type_hash, parse_bad_version) { const std::string test_value = - "RIHS213_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + "RIHS02_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); - ASSERT_EQ(RCUTILS_RET_OK, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); - EXPECT_EQ(213, hash.version); + EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + rcutils_reset_error(); } From eb127f21a7f58b142c7c70fd11e6150deac0e553 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Mar 2023 14:57:48 -0800 Subject: [PATCH 54/65] Rename rosidl_generator_type_hash to rosidl_generator_type_description Signed-off-by: Emerson Knapp --- README.md | 4 ++-- rosidl_generator_c/cmake/register_c.cmake | 2 +- .../rosidl_generator_c_generate_interfaces.cmake | 2 +- rosidl_generator_c/package.xml | 4 ++-- rosidl_generator_cpp/cmake/register_cpp.cmake | 2 +- ...osidl_generator_cpp_generate_interfaces.cmake | 2 +- rosidl_generator_cpp/package.xml | 2 +- rosidl_generator_tests/CMakeLists.txt | 6 +++--- rosidl_generator_tests/package.xml | 2 +- .../test_type_hash.py | 4 ++-- .../CMakeLists.txt | 2 +- .../README.md | 2 +- .../bin/rosidl_generator_type_description | 16 ++++++++-------- ...or_type_description_generate_interfaces.cmake | 12 ++++++------ .../package.xml | 4 ++-- .../resource/HashedTypeDescription.schema.json | 0 ...dl_generator_type_description-extras.cmake.in | 16 ++++++++++++++++ .../__init__.py | 0 .../test/test_serializers.py | 2 +- .../rosidl_generator_type_hash-extras.cmake.in | 16 ---------------- 20 files changed, 50 insertions(+), 50 deletions(-) rename rosidl_generator_tests/test/{rosidl_generator_type_hash => rosidl_generator_type_description}/test_type_hash.py (92%) rename {rosidl_generator_type_hash => rosidl_generator_type_description}/CMakeLists.txt (93%) rename {rosidl_generator_type_hash => rosidl_generator_type_description}/README.md (96%) rename rosidl_generator_type_hash/bin/rosidl_generator_type_hash => rosidl_generator_type_description/bin/rosidl_generator_type_description (68%) rename rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake => rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake (93%) rename {rosidl_generator_type_hash => rosidl_generator_type_description}/package.xml (86%) rename {rosidl_generator_type_hash => rosidl_generator_type_description}/resource/HashedTypeDescription.schema.json (100%) create mode 100644 rosidl_generator_type_description/rosidl_generator_type_description-extras.cmake.in rename {rosidl_generator_type_hash/rosidl_generator_type_hash => rosidl_generator_type_description/rosidl_generator_type_description}/__init__.py (100%) rename {rosidl_generator_type_hash => rosidl_generator_type_description}/test/test_serializers.py (96%) delete mode 100644 rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in diff --git a/README.md b/README.md index b556c1249..e4f7d18b0 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ See [documentation](https://docs.ros.org/en/rolling/Concepts/About-Internal-Inte * Generate the ROS interfaces in C * [rosidl_generator_cpp](./rosidl_generator_cpp) * Generate the ROS interfaces in C++ -* [rosidl_generator_type_hash](./rosidl_generator_type_hash) - * Generate SHA256 hash values for ROS 2 interface descriptions for use by other generators +* [rosidl_generator_type_description](./rosidl_generator_type_desrciption) + * Generate SHA256 hash values and ROS 2 interface descriptions for use by other generators * [rosidl_parser](./rosidl_parser) * Parser for `.idl` ROS interface files * [rosidl_runtime_c](./rosidl_runtime_c) diff --git a/rosidl_generator_c/cmake/register_c.cmake b/rosidl_generator_c/cmake/register_c.cmake index 36b1f307a..7f6b84b17 100644 --- a/rosidl_generator_c/cmake/register_c.cmake +++ b/rosidl_generator_c/cmake/register_c.cmake @@ -14,7 +14,7 @@ macro(rosidl_generator_c_extras BIN GENERATOR_FILES TEMPLATE_DIR) find_package(ament_cmake_core QUIET REQUIRED) - find_package(rosidl_generator_type_hash QUIET REQUIRED) + find_package(rosidl_generator_type_description QUIET REQUIRED) ament_register_extension( "rosidl_generate_idl_interfaces" "rosidl_generator_c" diff --git a/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake b/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake index 7d44991a4..f38ab14a2 100644 --- a/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake +++ b/rosidl_generator_c/cmake/rosidl_generator_c_generate_interfaces.cmake @@ -145,7 +145,7 @@ target_link_libraries(${rosidl_generate_interfaces_TARGET}${_target_suffix} PUBL rcutils::rcutils) add_dependencies( ${rosidl_generate_interfaces_TARGET}${_target_suffix} - ${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_hash) + ${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_description) # Make top level generation target depend on this generated library add_dependencies( diff --git a/rosidl_generator_c/package.xml b/rosidl_generator_c/package.xml index 0d1cad9e6..4a9d4765a 100644 --- a/rosidl_generator_c/package.xml +++ b/rosidl_generator_c/package.xml @@ -24,7 +24,7 @@ python3 rosidl_pycommon - rosidl_generator_type_hash + rosidl_generator_type_description rosidl_typesupport_interface rcutils @@ -32,7 +32,7 @@ rosidl_cli rosidl_parser rcutils - rosidl_generator_type_hash + rosidl_generator_type_description ament_lint_auto ament_lint_common diff --git a/rosidl_generator_cpp/cmake/register_cpp.cmake b/rosidl_generator_cpp/cmake/register_cpp.cmake index afdf2c4ec..6c0eaba86 100644 --- a/rosidl_generator_cpp/cmake/register_cpp.cmake +++ b/rosidl_generator_cpp/cmake/register_cpp.cmake @@ -14,7 +14,7 @@ macro(rosidl_generator_cpp_extras BIN GENERATOR_FILES TEMPLATE_DIR) find_package(ament_cmake_core QUIET REQUIRED) - find_package(rosidl_generator_type_hash QUIET REQUIRED) + find_package(rosidl_generator_type_description QUIET REQUIRED) ament_register_extension( "rosidl_generate_idl_interfaces" "rosidl_generator_cpp" diff --git a/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake b/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake index 3046a8e70..526ce76c2 100644 --- a/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake +++ b/rosidl_generator_cpp/cmake/rosidl_generator_cpp_generate_interfaces.cmake @@ -102,7 +102,7 @@ add_custom_target( ) add_dependencies( ${rosidl_generate_interfaces_TARGET}__cpp - ${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_hash) + ${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_description) set(_target_suffix "__rosidl_generator_cpp") add_library(${rosidl_generate_interfaces_TARGET}${_target_suffix} INTERFACE) diff --git a/rosidl_generator_cpp/package.xml b/rosidl_generator_cpp/package.xml index 51c31730e..34900b24e 100644 --- a/rosidl_generator_cpp/package.xml +++ b/rosidl_generator_cpp/package.xml @@ -28,7 +28,7 @@ ament_index_python rosidl_cli - rosidl_generator_type_hash + rosidl_generator_type_description rosidl_parser ament_lint_auto diff --git a/rosidl_generator_tests/CMakeLists.txt b/rosidl_generator_tests/CMakeLists.txt index 4f6ffc44d..a8746bc83 100644 --- a/rosidl_generator_tests/CMakeLists.txt +++ b/rosidl_generator_tests/CMakeLists.txt @@ -23,7 +23,7 @@ if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) find_package(rosidl_cmake REQUIRED) find_package(rosidl_generator_cpp REQUIRED) - find_package(rosidl_generator_type_hash REQUIRED) + find_package(rosidl_generator_type_description REQUIRED) find_package(rosidl_runtime_c REQUIRED) find_package(rosidl_runtime_cpp REQUIRED) find_package(test_interface_files REQUIRED) @@ -112,8 +112,8 @@ if(BUILD_TESTING) rosidl_runtime_c::rosidl_runtime_c ) - ament_add_pytest_test(test_hash_generator test/rosidl_generator_type_hash - ENV GENERATED_TEST_FILE_DIR=${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash/${PROJECT_NAME} + ament_add_pytest_test(test_hash_generator test/rosidl_generator_type_description + ENV GENERATED_TEST_FILE_DIR=${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_description/${PROJECT_NAME} ) endif() diff --git a/rosidl_generator_tests/package.xml b/rosidl_generator_tests/package.xml index be0f1d683..9ee03be32 100644 --- a/rosidl_generator_tests/package.xml +++ b/rosidl_generator_tests/package.xml @@ -27,7 +27,7 @@ rosidl_cmake rosidl_generator_c rosidl_generator_cpp - rosidl_generator_type_hash + rosidl_generator_type_description rosidl_runtime_c rosidl_runtime_cpp test_interface_files diff --git a/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py b/rosidl_generator_tests/test/rosidl_generator_type_description/test_type_hash.py similarity index 92% rename from rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py rename to rosidl_generator_tests/test/rosidl_generator_type_description/test_type_hash.py index 4adf49013..4fde93d05 100644 --- a/rosidl_generator_tests/test/rosidl_generator_type_hash/test_type_hash.py +++ b/rosidl_generator_tests/test/rosidl_generator_type_description/test_type_hash.py @@ -21,9 +21,9 @@ def test_type_hash(): - """Test all rosidl_generator_type_hash output files against defined schemas.""" + """Test all rosidl_generator_type_description output files against defined schemas.""" schema_path = ( - Path(get_package_share_directory('rosidl_generator_type_hash')) / 'resource' / + Path(get_package_share_directory('rosidl_generator_type_description')) / 'resource' / 'HashedTypeDescription.schema.json') with schema_path.open('r') as schema_file: schema = json.load(schema_file) diff --git a/rosidl_generator_type_hash/CMakeLists.txt b/rosidl_generator_type_description/CMakeLists.txt similarity index 93% rename from rosidl_generator_type_hash/CMakeLists.txt rename to rosidl_generator_type_description/CMakeLists.txt index 5e95030be..2a3949a92 100644 --- a/rosidl_generator_type_hash/CMakeLists.txt +++ b/rosidl_generator_type_description/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.12) -project(rosidl_generator_type_hash) +project(rosidl_generator_type_description) find_package(ament_cmake_python REQUIRED) find_package(ament_cmake_ros REQUIRED) diff --git a/rosidl_generator_type_hash/README.md b/rosidl_generator_type_description/README.md similarity index 96% rename from rosidl_generator_type_hash/README.md rename to rosidl_generator_type_description/README.md index 3d1bdb950..318d2a05d 100644 --- a/rosidl_generator_type_hash/README.md +++ b/rosidl_generator_type_description/README.md @@ -1,4 +1,4 @@ -# rosidl_generator_type_hash +# rosidl_generator_type_description This generator serializes ROS 2 interface descriptions (message, service, action) to a common format and uses SHA256 to hash that representation into a unique hash for each type. diff --git a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash b/rosidl_generator_type_description/bin/rosidl_generator_type_description similarity index 68% rename from rosidl_generator_type_hash/bin/rosidl_generator_type_hash rename to rosidl_generator_type_description/bin/rosidl_generator_type_description index fbf92e874..6c0175e37 100755 --- a/rosidl_generator_type_hash/bin/rosidl_generator_type_hash +++ b/rosidl_generator_type_description/bin/rosidl_generator_type_description @@ -18,20 +18,20 @@ import os import sys try: - from rosidl_generator_type_hash import generate_type_hash + from rosidl_generator_type_description import generate_type_hash except ImportError: # modifying sys.path and importing the Python package with the same # name as this script does not work on Windows - rosidl_generator_type_hash_root = os.path.dirname(os.path.dirname(__file__)) - rosidl_generator_type_hash_module = os.path.join( - rosidl_generator_type_hash_root, 'rosidl_generator_type_hash', '__init__.py') - if not os.path.exists(rosidl_generator_type_hash_module): + rosidl_generator_type_description_root = os.path.dirname(os.path.dirname(__file__)) + rosidl_generator_type_description_module = os.path.join( + rosidl_generator_type_description_root, 'rosidl_generator_type_description', '__init__.py') + if not os.path.exists(rosidl_generator_type_description_module): raise from importlib.machinery import SourceFileLoader - loader = SourceFileLoader('rosidl_generator_type_hash', rosidl_generator_type_hash_module) - rosidl_generator_type_hash = loader.load_module() - generate_type_hash = rosidl_generator_type_hash.generate_type_hash + loader = SourceFileLoader('rosidl_generator_type_description', rosidl_generator_type_description_module) + rosidl_generator_type_description = loader.load_module() + generate_type_hash = rosidl_generator_type_description.generate_type_hash def main(argv=sys.argv[1:]): diff --git a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake b/rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake similarity index 93% rename from rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake rename to rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake index d9bd2a24b..253948b50 100644 --- a/rosidl_generator_type_hash/cmake/rosidl_generator_type_hash_generate_interfaces.cmake +++ b/rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake @@ -14,7 +14,7 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) -set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash/${PROJECT_NAME}") +set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_description/${PROJECT_NAME}") set(_generated_files "") set(_generated_tuples "") @@ -42,8 +42,8 @@ set(${rosidl_generate_interfaces_TARGET}__HASH_TUPLES ${_generated_tuples}) # Validate that all dependencies exist set(target_dependencies - "${rosidl_generator_type_hash_BIN}" - ${rosidl_generator_type_hash_GENERATOR_FILES} + "${rosidl_generator_type_description_BIN}" + ${rosidl_generator_type_description_GENERATOR_FILES} ${rosidl_generate_interfaces_ABS_IDL_FILES} ${_dependency_files}) foreach(dep ${target_dependencies}) @@ -52,7 +52,7 @@ foreach(dep ${target_dependencies}) endif() endforeach() -set(_generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_hash__arguments.json") +set(_generator_arguments_file "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_type_description__arguments.json") rosidl_write_generator_arguments( "${_generator_arguments_file}" PACKAGE_NAME "${PROJECT_NAME}" @@ -65,7 +65,7 @@ rosidl_write_generator_arguments( add_custom_command( COMMAND Python3::Interpreter ARGS - ${rosidl_generator_type_hash_BIN} + ${rosidl_generator_type_description_BIN} --generator-arguments-file "${_generator_arguments_file}" OUTPUT ${_generated_files} DEPENDS ${target_dependencies} @@ -73,7 +73,7 @@ add_custom_command( VERBATIM ) -set(_target "${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_hash") +set(_target "${rosidl_generate_interfaces_TARGET}__rosidl_generator_type_description") add_custom_target(${_target} DEPENDS ${_generated_files}) # Make top level generation target depend on this generated library diff --git a/rosidl_generator_type_hash/package.xml b/rosidl_generator_type_description/package.xml similarity index 86% rename from rosidl_generator_type_hash/package.xml rename to rosidl_generator_type_description/package.xml index c00156a87..88f87f978 100644 --- a/rosidl_generator_type_hash/package.xml +++ b/rosidl_generator_type_description/package.xml @@ -1,9 +1,9 @@ - rosidl_generator_type_hash + rosidl_generator_type_description 3.3.1 - Generate REP-2011 type version hashes for ROS interfaces. + Generate hashes and descriptions of ROS 2 interface types, per REP-2011. Emerson Knapp diff --git a/rosidl_generator_type_hash/resource/HashedTypeDescription.schema.json b/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json similarity index 100% rename from rosidl_generator_type_hash/resource/HashedTypeDescription.schema.json rename to rosidl_generator_type_description/resource/HashedTypeDescription.schema.json diff --git a/rosidl_generator_type_description/rosidl_generator_type_description-extras.cmake.in b/rosidl_generator_type_description/rosidl_generator_type_description-extras.cmake.in new file mode 100644 index 000000000..eefe59a1b --- /dev/null +++ b/rosidl_generator_type_description/rosidl_generator_type_description-extras.cmake.in @@ -0,0 +1,16 @@ +find_package(ament_cmake_core QUIET REQUIRED) + +ament_register_extension( + "rosidl_generate_idl_interfaces" + "rosidl_generator_type_description" + "rosidl_generator_type_description_generate_interfaces.cmake") + +set(rosidl_generator_type_description_BIN + "${rosidl_generator_type_description_DIR}/../../../lib/rosidl_generator_type_description/rosidl_generator_type_description") +normalize_path(rosidl_generator_type_description_BIN + "${rosidl_generator_type_description_BIN}") + +set(rosidl_generator_type_description_GENERATOR_FILES + "${rosidl_generator_type_description_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_description/__init__.py") +normalize_path(rosidl_generator_type_description_GENERATOR_FILES + "${rosidl_generator_type_description_GENERATOR_FILES}") diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py similarity index 100% rename from rosidl_generator_type_hash/rosidl_generator_type_hash/__init__.py rename to rosidl_generator_type_description/rosidl_generator_type_description/__init__.py diff --git a/rosidl_generator_type_hash/test/test_serializers.py b/rosidl_generator_type_description/test/test_serializers.py similarity index 96% rename from rosidl_generator_type_hash/test/test_serializers.py rename to rosidl_generator_type_description/test/test_serializers.py index 74654cafa..d50f96b4d 100644 --- a/rosidl_generator_type_hash/test/test_serializers.py +++ b/rosidl_generator_type_description/test/test_serializers.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from rosidl_generator_type_hash import serialize_field_type +from rosidl_generator_type_description import serialize_field_type from rosidl_parser import definition diff --git a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in b/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in deleted file mode 100644 index 2dc1d2e96..000000000 --- a/rosidl_generator_type_hash/rosidl_generator_type_hash-extras.cmake.in +++ /dev/null @@ -1,16 +0,0 @@ -find_package(ament_cmake_core QUIET REQUIRED) - -ament_register_extension( - "rosidl_generate_idl_interfaces" - "rosidl_generator_type_hash" - "rosidl_generator_type_hash_generate_interfaces.cmake") - -set(rosidl_generator_type_hash_BIN - "${rosidl_generator_type_hash_DIR}/../../../lib/rosidl_generator_type_hash/rosidl_generator_type_hash") -normalize_path(rosidl_generator_type_hash_BIN - "${rosidl_generator_type_hash_BIN}") - -set(rosidl_generator_type_hash_GENERATOR_FILES - "${rosidl_generator_type_hash_DIR}/../../../@PYTHON_INSTALL_DIR@/rosidl_generator_type_hash/__init__.py") -normalize_path(rosidl_generator_type_hash_GENERATOR_FILES - "${rosidl_generator_type_hash_GENERATOR_FILES}") From 3fce7b596af6e63233159554fc595a2db46e01af Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Tue, 7 Mar 2023 16:52:55 -0800 Subject: [PATCH 55/65] Address review comments Signed-off-by: Emerson Knapp --- .../rosidl_generator_c/__init__.py | 14 +- .../HashedTypeDescription.schema.json | 6 +- .../__init__.py | 23 +++- rosidl_runtime_c/CMakeLists.txt | 1 - .../include/rosidl_runtime_c/type_hash.h | 2 + rosidl_runtime_c/src/type_hash.c | 127 +++++++++++------- rosidl_runtime_c/test/test_type_hash.cpp | 1 + 7 files changed, 112 insertions(+), 62 deletions(-) diff --git a/rosidl_generator_c/rosidl_generator_c/__init__.py b/rosidl_generator_c/rosidl_generator_c/__init__.py index a1454c1d0..9f795a8c0 100644 --- a/rosidl_generator_c/rosidl_generator_c/__init__.py +++ b/rosidl_generator_c/rosidl_generator_c/__init__.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - +from rosidl_generator_type_description import parse_rihs_string +from rosidl_generator_type_description import RIHS01_HASH_VALUE_SIZE from rosidl_parser.definition import AbstractGenericString from rosidl_parser.definition import AbstractSequence from rosidl_parser.definition import AbstractString @@ -225,18 +225,14 @@ def escape_wstring(s): def type_hash_to_c_definition(variable_name, hash_string, indent=0): """Generate empy for rosidl_type_hash_t instance with 8 bytes per line for readability.""" - hash_length = 32 bytes_per_line = 8 indent_str = ' ' * (indent + 2) - pattern = re.compile(r'RIHS([0-9a-f]{2})_([0-9a-f]{64})') - match = pattern.match(hash_string) - if not match: - raise Exception('Type hash string does not match expected RIHS format') - version, value = match.group(1, 2) + version, value = parse_rihs_string(hash_string) + assert version == 1, 'This function only knows how to generate RIHS01 definitions.' result = f'rosidl_type_hash_t {variable_name} = {{{version}, {{' - for i in range(hash_length): + for i in range(RIHS01_HASH_VALUE_SIZE): if i % bytes_per_line == 0: result += f'\n{indent_str} ' result += f'0x{value[i * 2:i * 2 + 2]},' diff --git a/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json b/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json index e1b9d05db..1563ff285 100644 --- a/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json +++ b/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "HashedTypeDescription.schema.json", "title": "HashedTypeDescription", - "description": "TODO", + "description": "Contains hashes and full type description for a ROS 2 interface. TypeDescription, IndividualTypeDescription, Field, and FieldType are exact representations of type_description_interfaces/msg types, see their .msg files for semantic comments.", "type": "object", "properties": { "hashes": { @@ -66,11 +66,11 @@ }, "TypeDescription": { "type": "object", - "$comment": "All whitespace must be excluded for consistent hashing, which this schema cannot enforce.", + "$comment": "For hashing: All whitespace must be excluded, which this schema cannot enforce.", "properties": { "type_description": {"$ref": "#/$defs/IndividualTypeDescription"}, "referenced_type_descriptions": { - "$comment": "Referenced type descriptions must be alphabetized, which this schema cannot enforce.", + "$comment": "For hashing: Referenced type descriptions must be alphabetized, which this schema cannot enforce.", "type": "array", "items": { "$ref": "#/$defs/IndividualTypeDescription" } } diff --git a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py index 73aeea0ea..ef7e0da10 100644 --- a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py +++ b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py @@ -15,14 +15,21 @@ import hashlib import json from pathlib import Path +import re import sys from typing import List, Tuple from rosidl_parser import definition from rosidl_parser.parser import parse_idl_file -# ROS Interface Hashing Standard, per REP-2011 -RIHS_VERSION = '01' +# RIHS: ROS Interface Hashing Standard, per REP-2011 +# NOTE: These values and implementations must be updated if +# - type_description_interfaces messsages change, or +# - the hashing algorithm for type descriptions changes +# Both changes require an increment of the RIHS version +RIHS01_PREFIX = 'RIHS01_' +RIHS01_HASH_VALUE_SIZE = 32 +RIHS01_PATTERN = re.compile(r'RIHS([0-9a-f]{2})_([0-9a-f]{64})') def generate_type_hash(generator_arguments_file: str) -> List[str]: @@ -72,6 +79,15 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: return generated_files +def parse_rihs_string(rihs_str: str) -> Tuple[int, str]: + """Parse RIHS string, return (version, value) tuple.""" + match = RIHS01_PATTERN.match(rihs_str) + if not match: + raise ValueError(f'Type hash string {rihs_str} does not match expected RIHS format.') + version, value = match.group(1, 2) + return (int(version), value) + + # This mapping must match the constants defined in type_description_interfaces/msgs/FieldType.msg # NOTE: Nonexplicit integer types are not defined in FieldType (short, long, long long). # If a ROS IDL uses these, this generator will throw a KeyError. @@ -446,10 +462,9 @@ def _calculate_hash_tree(self) -> dict: separators=(',', ': '), sort_keys=False ) - prefix = f'RIHS{RIHS_VERSION}_' sha = hashlib.sha256() sha.update(hashable_repr.encode('utf-8')) - type_hash = prefix + sha.hexdigest() + type_hash = RIHS01_PREFIX + sha.hexdigest() type_hash_infos = { self.interface_type: type_hash, diff --git a/rosidl_runtime_c/CMakeLists.txt b/rosidl_runtime_c/CMakeLists.txt index 202a76f7e..5da904f1c 100644 --- a/rosidl_runtime_c/CMakeLists.txt +++ b/rosidl_runtime_c/CMakeLists.txt @@ -30,7 +30,6 @@ target_include_directories(${PROJECT_NAME} PUBLIC ament_target_dependencies(${PROJECT_NAME} "rcutils" "rosidl_typesupport_interface") -target_link_libraries(${PROJECT_NAME} m) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang") set_target_properties(${PROJECT_NAME} PROPERTIES COMPILE_OPTIONS -Wall -Wextra -Wpedantic) diff --git a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h index 7fa66893a..dfe4c516e 100644 --- a/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h +++ b/rosidl_runtime_c/include/rosidl_runtime_c/type_hash.h @@ -68,6 +68,8 @@ rosidl_stringify_type_hash( /** * \param[in] type_hash_string Null-terminated string with the hash representation * \param[out] hash_out Preallocated structure to be filled with parsed hash information. + * hash_out->version will be 0 if no version could be parsed, + * but if a version could be determined this field will be set even if an error is returned * \return RCTUILS_RET_INVALID_ARGUMENT on any null pointer argumunts, or malformed hash string. * \return RCUTILS_RET_OK otherwise */ diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index 46cba9fbf..ba60474ce 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -12,12 +12,51 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include "rosidl_runtime_c/type_hash.h" #include "rcutils/error_handling.h" -#include "rcutils/format_string.h" + +static const char RIHS01_PREFIX[] = "RIHS01_"; +// Hash representation is hex string, two characters per byte +static const size_t RIHS_VERSION_IDX = 4; +static const size_t RIHS_PREFIX_LEN = 7; +static const size_t RIHS01_STRING_LEN = RIHS_PREFIX_LEN + (ROSIDL_TYPE_HASH_SIZE * 2); + +static bool _ishexdigit(char c) +{ + return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); +} + +/// Translate a single character hex digit to a nibble +static uint8_t _xatoi(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 0xa; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 0xa; + } + return -1; +} + +/// Tranlate a byte value to two hex characters +static void _xitoa(uint8_t val, char * dest) +{ + uint8_t nibble = 0; + for (size_t n = 0; n < 2; n++) { + nibble = (val >> (4 * n)) & 0x0f; + if (nibble < 0xa) { + dest[n] = '0' + nibble; + } else { // 0xa <= nibble < 0x10 + dest[n] = 'a' + (nibble - 0xa); + } + } +} rosidl_type_hash_t rosidl_get_zero_initialized_type_hash(void) @@ -32,55 +71,29 @@ rosidl_stringify_type_hash( rcutils_allocator_t allocator, char ** output_string) { - if (!type_hash) { - RCUTILS_SET_ERROR_MSG("Null type_hash argument."); - return RCUTILS_RET_INVALID_ARGUMENT; - } + RCUTILS_CHECK_ARGUMENT_FOR_NULL(type_hash, RCUTILS_RET_INVALID_ARGUMENT); if (!rcutils_allocator_is_valid(&allocator)) { RCUTILS_SET_ERROR_MSG("Invalid allocator"); return RCUTILS_RET_INVALID_ARGUMENT; } - if (!output_string) { - RCUTILS_SET_ERROR_MSG("Null string destination pointer."); - return RCUTILS_RET_INVALID_ARGUMENT; - } + RCUTILS_CHECK_ARGUMENT_FOR_NULL(output_string, RCUTILS_RET_INVALID_ARGUMENT); - // Hash representation is hex string, two characters per byte - const char * fmt = "RIHS%02d_%64d"; - const size_t prefix_len = strlen("RIHS01_"); - char * local_output = rcutils_format_string(allocator, fmt, type_hash->version, 0); + char * local_output = allocator.allocate(RIHS01_STRING_LEN + 1, allocator.state); if (!local_output) { *output_string = NULL; - RCUTILS_SET_ERROR_MSG("Unable to allocate space for stringified type hash."); + RCUTILS_SET_ERROR_MSG("Unable to allocate space for type hash string."); return RCUTILS_RET_BAD_ALLOC; } + local_output[RIHS01_STRING_LEN] = '\0'; + memcpy(local_output, RIHS01_PREFIX, RIHS_PREFIX_LEN); for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { - snprintf(local_output + prefix_len + (i * 2), 3, "%02x", type_hash->value[i]); // NOLINT + _xitoa(type_hash->value[i], local_output + RIHS_PREFIX_LEN + (i * 2)); } *output_string = local_output; return RCUTILS_RET_OK; } -static int _xatoi(char c) -{ - if (c >= '0' && c <= '9') { - return c - '0'; - } - if (c >= 'A' && c <= 'F') { - return c - 'A' + 0xa; - } - if (c >= 'a' && c <= 'f') { - return c - 'a' + 0xa; - } - return -1; -} - -static int _str_to_byte(const char * str) -{ - return (_xatoi(str[0]) << 4) + _xatoi(str[1]); -} - rcutils_ret_t rosidl_parse_type_hash_string( const char * type_hash_string, @@ -88,22 +101,46 @@ rosidl_parse_type_hash_string( { RCUTILS_CHECK_ARGUMENT_FOR_NULL(type_hash_string, RCUTILS_RET_INVALID_ARGUMENT); RCUTILS_CHECK_ARGUMENT_FOR_NULL(hash_out, RCUTILS_RET_INVALID_ARGUMENT); - static const size_t kprefix_len = sizeof("RIHS01_") - 1; - static const size_t kvalue_len = 64; hash_out->version = 0; + size_t input_len = strlen(type_hash_string); - if (strlen(type_hash_string) != (kprefix_len + kvalue_len)) { - RCUTILS_SET_ERROR_MSG("Hash string incorrect size."); + // Check prefix + if (input_len < RIHS_PREFIX_LEN) { + RCUTILS_SET_ERROR_MSG("Hash string not long enough to contain RIHS prefix."); return RCUTILS_RET_INVALID_ARGUMENT; } - if (0 != strncmp(type_hash_string, "RIHS01_", 7)) { - RCUTILS_SET_ERROR_MSG("Type hash string is not prefixed RIHS01_"); + if (0 != strncmp(type_hash_string, RIHS01_PREFIX, RIHS_VERSION_IDX)) { + RCUTILS_SET_ERROR_MSG("Hash string doesn't start with RIHS."); return RCUTILS_RET_INVALID_ARGUMENT; } - hash_out->version = 1; - const char * value_str = type_hash_string + kprefix_len; - for (size_t i = 0; i < 32; i++) { - int byte_val = _str_to_byte(value_str + (i * 2)); + + // Parse version + char version_top = type_hash_string[RIHS_VERSION_IDX]; + char version_bot = type_hash_string[RIHS_VERSION_IDX + 1]; + if (!(_ishexdigit(version_top) && _ishexdigit(version_bot))) { + RCUTILS_SET_ERROR_MSG("RIHS version is not a 2-digit hex string."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + hash_out->version = (_xatoi(version_top) << 4) + _xatoi(version_bot); + + if (hash_out->version != 1) { + RCUTILS_SET_ERROR_MSG("Do not know how to parse RIHS version."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + if (input_len != RIHS01_STRING_LEN) { + RCUTILS_SET_ERROR_MSG("RIHS string is the incorrect size to contain a RIHS01 value."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + const char * value_str = type_hash_string + RIHS_PREFIX_LEN; + for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { + if (!_ishexdigit(value_str[i * 2])) { + RCUTILS_SET_ERROR_MSG("Type hash string value contained non-hex-digit character."); + return RCUTILS_RET_INVALID_ARGUMENT; + } + } + for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { + // No error checking on byte values because the whole string was checked in prior loop + uint8_t byte_val = (_xatoi(value_str[i * 2]) << 4) + _xatoi(value_str[i * 2 + 1]); if (byte_val < 0) { RCUTILS_SET_ERROR_MSG("Type hash string value did not contain only hex digits."); return RCUTILS_RET_INVALID_ARGUMENT; diff --git a/rosidl_runtime_c/test/test_type_hash.cpp b/rosidl_runtime_c/test/test_type_hash.cpp index 8d68a2da2..6c785ce9b 100644 --- a/rosidl_runtime_c/test/test_type_hash.cpp +++ b/rosidl_runtime_c/test/test_type_hash.cpp @@ -92,5 +92,6 @@ TEST(type_hash, parse_bad_version) { "RIHS02_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + EXPECT_EQ(hash.version, 2); rcutils_reset_error(); } From 0b0886258b05d5e4f8ae1df85376d5dcca77d110 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Mar 2023 12:23:55 -0800 Subject: [PATCH 56/65] Go full-constant for windows complaints Signed-off-by: Emerson Knapp --- rosidl_runtime_c/src/type_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index ba60474ce..fccfc3fc2 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -22,7 +22,7 @@ static const char RIHS01_PREFIX[] = "RIHS01_"; // Hash representation is hex string, two characters per byte static const size_t RIHS_VERSION_IDX = 4; static const size_t RIHS_PREFIX_LEN = 7; -static const size_t RIHS01_STRING_LEN = RIHS_PREFIX_LEN + (ROSIDL_TYPE_HASH_SIZE * 2); +static const size_t RIHS01_STRING_LEN = 71; // RIHS_PREFIX_LEN + (ROSIDL_TYPE_HASH_SIZE * 2); static bool _ishexdigit(char c) { From db50587cd2336929ae0b87b0b7be3e16d1ac6ecd Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Mar 2023 13:36:31 -0800 Subject: [PATCH 57/65] Fix type hash validation error and stringify-reversing error, add tests for them Signed-off-by: Emerson Knapp --- rosidl_runtime_c/src/type_hash.c | 8 ++++---- rosidl_runtime_c/test/test_type_hash.cpp | 19 ++++++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index fccfc3fc2..b641d96f0 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -49,7 +49,7 @@ static void _xitoa(uint8_t val, char * dest) { uint8_t nibble = 0; for (size_t n = 0; n < 2; n++) { - nibble = (val >> (4 * n)) & 0x0f; + nibble = (val >> (4 * (1 - n))) & 0x0f; if (nibble < 0xa) { dest[n] = '0' + nibble; } else { // 0xa <= nibble < 0x10 @@ -132,15 +132,15 @@ rosidl_parse_type_hash_string( return RCUTILS_RET_INVALID_ARGUMENT; } const char * value_str = type_hash_string + RIHS_PREFIX_LEN; - for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { - if (!_ishexdigit(value_str[i * 2])) { + for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE * 2; i++) { + if (!_ishexdigit(value_str[i])) { RCUTILS_SET_ERROR_MSG("Type hash string value contained non-hex-digit character."); return RCUTILS_RET_INVALID_ARGUMENT; } } for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { // No error checking on byte values because the whole string was checked in prior loop - uint8_t byte_val = (_xatoi(value_str[i * 2]) << 4) + _xatoi(value_str[i * 2 + 1]); + uint8_t byte_val = (_xatoi(value_str[i * 2]) << 4) + _xatoi(value_str[(i * 2) + 1]); if (byte_val < 0) { RCUTILS_SET_ERROR_MSG("Type hash string value did not contain only hex digits."); return RCUTILS_RET_INVALID_ARGUMENT; diff --git a/rosidl_runtime_c/test/test_type_hash.cpp b/rosidl_runtime_c/test/test_type_hash.cpp index 6c785ce9b..e684efa8d 100644 --- a/rosidl_runtime_c/test/test_type_hash.cpp +++ b/rosidl_runtime_c/test/test_type_hash.cpp @@ -27,11 +27,11 @@ TEST(type_hash, init_zero_hash) { TEST(type_hash, stringify_basic) { const std::string expected = - "RIHS01_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + "RIHS01_000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); hash.version = 1; for (size_t i = 0; i < sizeof(hash.value); i++) { - hash.value[i] = (i % 0x10) * 0x10 + (i % 0x10); + hash.value[i] = i; } auto allocator = rcutils_get_default_allocator(); char * hash_string = nullptr; @@ -44,14 +44,14 @@ TEST(type_hash, stringify_basic) { TEST(type_hash, parse_basic) { const std::string test_value = - "RIHS01_00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + "RIHS01_000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); ASSERT_EQ(RCUTILS_RET_OK, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); EXPECT_EQ(1, hash.version); for (size_t i = 0; i < sizeof(hash.value); i++) { - size_t expected_value = (i % 0x10) * 0x10 + (i % 0x10); - EXPECT_EQ(expected_value, hash.value[i]); + size_t expected_value = i; + EXPECT_EQ(expected_value, hash.value[i]) << "At byte " << i; } } @@ -95,3 +95,12 @@ TEST(type_hash, parse_bad_version) { EXPECT_EQ(hash.version, 2); rcutils_reset_error(); } + +TEST(type_hash, parse_bad_value) { + const std::string test_value = + "RIHS01_00112233445566778899aabbccddgeff00112233445566778899aabbccddeeff"; + rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); + EXPECT_EQ(RCUTILS_RET_INVALID_ARGUMENT, rosidl_parse_type_hash_string(test_value.c_str(), &hash)); + EXPECT_EQ(hash.version, 1); + rcutils_reset_error(); +} From 1f2c6cadbde2930fd74190ef3cc129ea2eb19499 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Thu, 9 Mar 2023 16:27:15 -0800 Subject: [PATCH 58/65] Fix windows build: path handling fix, and type hash define initializer Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 10 +++++-- rosidl_generator_c/resource/msg__struct.h.em | 4 ++- .../rosidl_generator_c/__init__.py | 29 +++++++++++-------- .../resource/action__struct.hpp.em | 2 +- .../resource/msg__struct.hpp.em | 2 +- .../resource/srv__struct.hpp.em | 2 +- ...type_description_generate_interfaces.cmake | 5 ++-- rosidl_pycommon/rosidl_pycommon/__init__.py | 3 +- rosidl_runtime_c/test/test_type_hash.cpp | 2 +- .../resource/msg__type_support.c.em | 2 +- 10 files changed, 37 insertions(+), 24 deletions(-) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index c9b9d46ab..ecfd437fb 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -59,7 +59,10 @@ TEMPLATE( from rosidl_parser.definition import Service }@ @[for service in content.get_elements_of_type(Service)]@ -static const @(type_hash_to_c_definition(idl_structure_type_to_c_typename(service.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['service']))@ + +@{ hash_var = idl_structure_type_to_c_typename(service.namespaced_type) + '__TYPE_VERSION_HASH' }@ +#define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['service'], line_final_backslash=True)) +static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; @{ TEMPLATE( @@ -94,7 +97,10 @@ TEMPLATE( from rosidl_parser.definition import Action }@ @[for action in content.get_elements_of_type(Action)]@ -static const @(type_hash_to_c_definition(idl_structure_type_to_c_typename(action.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['action']))@ + +@{ hash_var = idl_structure_type_to_c_typename(action.namespaced_type) + '__TYPE_VERSION_HASH' }@ +#define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['action'], line_final_backslash=True)) +static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; @{ TEMPLATE( diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index c168c2c99..5f212b70a 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -180,7 +180,9 @@ typedef struct @(idl_structure_type_to_c_typename(message.structure.namespaced_t } @(idl_structure_type_to_c_typename(message.structure.namespaced_type)); // Type Version Hash for interface -static const @(type_hash_to_c_definition(idl_structure_type_to_c_typename(message.structure.namespaced_type) + "__TYPE_VERSION_HASH", type_hash['message']))@ +@{ hash_var = idl_structure_type_to_c_typename(message.structure.namespaced_type) + '__TYPE_VERSION_HASH' }@ +#define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['message'], line_final_backslash=True)) +static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/rosidl_generator_c/rosidl_generator_c/__init__.py b/rosidl_generator_c/rosidl_generator_c/__init__.py index 9f795a8c0..5e0f72d11 100644 --- a/rosidl_generator_c/rosidl_generator_c/__init__.py +++ b/rosidl_generator_c/rosidl_generator_c/__init__.py @@ -223,20 +223,25 @@ def escape_wstring(s): return escape_string(s) -def type_hash_to_c_definition(variable_name, hash_string, indent=0): +def type_hash_to_c_definition(hash_string, *, line_final_backslash=False): """Generate empy for rosidl_type_hash_t instance with 8 bytes per line for readability.""" - bytes_per_line = 8 - - indent_str = ' ' * (indent + 2) + bytes_per_row = 8 + rows = 4 + indent = 4 # Uncrustify prefers this indentation version, value = parse_rihs_string(hash_string) assert version == 1, 'This function only knows how to generate RIHS01 definitions.' - result = f'rosidl_type_hash_t {variable_name} = {{{version}, {{' - for i in range(RIHS01_HASH_VALUE_SIZE): - if i % bytes_per_line == 0: - result += f'\n{indent_str} ' - result += f'0x{value[i * 2:i * 2 + 2]},' - if i % bytes_per_line != bytes_per_line - 1: - result += ' ' - result += f'\n{indent_str}}}}};\n' + result = f'{{{version}, {{' + if line_final_backslash: + result += ' \\' + result += '\n' + for row in range(rows): + result += ' ' * (indent + 1) + for i in range(row * bytes_per_row, (row + 1) * bytes_per_row): + result += f' 0x{value[i * 2]}{value[i * 2 + 1]},' + if line_final_backslash: + result += ' \\' + result += '\n' + result += ' ' * indent + result += '}}' return result diff --git a/rosidl_generator_cpp/resource/action__struct.hpp.em b/rosidl_generator_cpp/resource/action__struct.hpp.em index 31dd44649..9c89bbdc8 100644 --- a/rosidl_generator_cpp/resource/action__struct.hpp.em +++ b/rosidl_generator_cpp/resource/action__struct.hpp.em @@ -79,7 +79,7 @@ namespace @(ns) @[end for]@ struct @(action.namespaced_type.name) { - static constexpr const @(type_hash_to_c_definition("TYPE_VERSION_HASH", type_hash['action'], indent=2))@ + static constexpr const rosidl_type_hash_t TYPE_VERSION_HASH = @(type_hash_to_c_definition(type_hash['action'])); /// The goal message defined in the action definition. using Goal = @(action_name)@(ACTION_GOAL_SUFFIX); diff --git a/rosidl_generator_cpp/resource/msg__struct.hpp.em b/rosidl_generator_cpp/resource/msg__struct.hpp.em index c1c144436..e66a922d3 100644 --- a/rosidl_generator_cpp/resource/msg__struct.hpp.em +++ b/rosidl_generator_cpp/resource/msg__struct.hpp.em @@ -103,7 +103,7 @@ struct @(message.structure.namespaced_type.name)_ using Type = @(message.structure.namespaced_type.name)_; // Type Version Hash for interface - constexpr static const @(type_hash_to_c_definition("TYPE_VERSION_HASH", type_hash['message'], indent=2))@ + constexpr static const rosidl_type_hash_t TYPE_VERSION_HASH = @(type_hash_to_c_definition(type_hash['message'])); @{ # The creation of the constructors for messages is a bit complicated. The goal diff --git a/rosidl_generator_cpp/resource/srv__struct.hpp.em b/rosidl_generator_cpp/resource/srv__struct.hpp.em index 8a9f6c4b2..54bc65993 100644 --- a/rosidl_generator_cpp/resource/srv__struct.hpp.em +++ b/rosidl_generator_cpp/resource/srv__struct.hpp.em @@ -37,7 +37,7 @@ struct @(service.namespaced_type.name) @{ service_typename = '::'.join(service.namespaced_type.namespaced_name()) }@ - static constexpr const @(type_hash_to_c_definition("TYPE_VERSION_HASH", type_hash['service'], indent=2))@ + static constexpr const rosidl_type_hash_t TYPE_VERSION_HASH = @(type_hash_to_c_definition(type_hash['service'])); using Request = @(service_typename)_Request; using Response = @(service_typename)_Response; diff --git a/rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake b/rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake index 253948b50..2cf1c7635 100644 --- a/rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake +++ b/rosidl_generator_type_description/cmake/rosidl_generator_type_description_generate_interfaces.cmake @@ -22,10 +22,11 @@ set(_generated_tuples "") foreach(_abs_idl_file ${rosidl_generate_interfaces_ABS_IDL_FILES}) get_filename_component(_parent_folder "${_abs_idl_file}" DIRECTORY) get_filename_component(_parent_folder "${_parent_folder}" NAME) - get_filename_component(_idl_stem "${_abs_idl_file}" NAME_WE) + get_filename_component(_idl_name "${_abs_idl_file}" NAME) + get_filename_component(_idl_stem "${_idl_name}" NAME_WE) set(_json_file "${_output_path}/${_parent_folder}/${_idl_stem}.json") list(APPEND _generated_files "${_json_file}") - list(APPEND _generated_tuples "${_parent_folder}/${_idl_stem}:${_json_file}") + list(APPEND _generated_tuples "${_parent_folder}/${_idl_name}:${_json_file}") endforeach() # Find dependency packages' generated files diff --git a/rosidl_pycommon/rosidl_pycommon/__init__.py b/rosidl_pycommon/rosidl_pycommon/__init__.py index 536e574ba..6fc6a1fe6 100644 --- a/rosidl_pycommon/rosidl_pycommon/__init__.py +++ b/rosidl_pycommon/rosidl_pycommon/__init__.py @@ -75,9 +75,8 @@ def generate_files( locator = IdlLocator(*idl_parts) idl_rel_path = pathlib.Path(idl_parts[1]) - idl_rel_stem = idl_rel_path.with_suffix('') if type_hashes_provided: - type_hash_file = type_hash_files[str(idl_rel_stem)] + type_hash_file = type_hash_files[idl_parts[1]] with open(type_hash_file, 'r') as f: type_hash_infos = json.load(f)['hashes'] else: diff --git a/rosidl_runtime_c/test/test_type_hash.cpp b/rosidl_runtime_c/test/test_type_hash.cpp index e684efa8d..8183ea41d 100644 --- a/rosidl_runtime_c/test/test_type_hash.cpp +++ b/rosidl_runtime_c/test/test_type_hash.cpp @@ -30,7 +30,7 @@ TEST(type_hash, stringify_basic) { "RIHS01_000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"; rosidl_type_hash_t hash = rosidl_get_zero_initialized_type_hash(); hash.version = 1; - for (size_t i = 0; i < sizeof(hash.value); i++) { + for (uint8_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { hash.value[i] = i; } auto allocator = rcutils_get_default_allocator(); diff --git a/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em b/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em index 58c48b813..1e8beda4f 100644 --- a/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em +++ b/rosidl_typesupport_introspection_c/resource/msg__type_support.c.em @@ -270,7 +270,7 @@ for index, member in enumerate(message.structure.members): static const rosidl_typesupport_introspection_c__MessageMembers @(function_prefix)__@(message.structure.namespaced_type.name)_message_members = { "@('__'.join([package_name] + list(interface_path.parents[0].parts)))", // message namespace "@(message.structure.namespaced_type.name)", // message name - @('__'.join(message.structure.namespaced_type.namespaced_name()))__TYPE_VERSION_HASH, + @('__'.join(message.structure.namespaced_type.namespaced_name()))__TYPE_VERSION_HASH__INIT, @(len(message.structure.members)), // number of fields sizeof(@('__'.join([package_name] + list(interface_path.parents[0].parts) + [message.structure.namespaced_type.name]))), @(function_prefix)__@(message.structure.namespaced_type.name)_message_member_array, // message members From 7d291cc232b7aa50103908a15e8830e4fbb5e26d Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Fri, 10 Mar 2023 00:41:45 -0800 Subject: [PATCH 59/65] Windows paths have colons Signed-off-by: Emerson Knapp --- .../rosidl_generator_type_description/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py index ef7e0da10..fe3de28ca 100644 --- a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py +++ b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py @@ -45,7 +45,7 @@ def generate_type_hash(generator_arguments_file: str) -> List[str]: package_name: output_dir } for include_tuple in include_paths: - include_parts = include_tuple.rsplit(':', 1) + include_parts = include_tuple.split(':', 1) assert len(include_parts) == 2 include_package_name, include_base_path = include_parts include_map[include_package_name] = Path(include_base_path) From f6571acf01002a33d7877fad3b14fe38d0fe5fdf Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Fri, 10 Mar 2023 11:33:42 -0800 Subject: [PATCH 60/65] Fix linter error by adding a futureproofing check Signed-off-by: Emerson Knapp --- rosidl_generator_c/rosidl_generator_c/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rosidl_generator_c/rosidl_generator_c/__init__.py b/rosidl_generator_c/rosidl_generator_c/__init__.py index 5e0f72d11..9be5d720e 100644 --- a/rosidl_generator_c/rosidl_generator_c/__init__.py +++ b/rosidl_generator_c/rosidl_generator_c/__init__.py @@ -227,6 +227,7 @@ def type_hash_to_c_definition(hash_string, *, line_final_backslash=False): """Generate empy for rosidl_type_hash_t instance with 8 bytes per line for readability.""" bytes_per_row = 8 rows = 4 + assert bytes_per_row * rows == RIHS01_HASH_VALUE_SIZE, 'This function is outdated.' indent = 4 # Uncrustify prefers this indentation version, value = parse_rihs_string(hash_string) assert version == 1, 'This function only knows how to generate RIHS01 definitions.' From fa9bf550ee96b0229558b9bb0fe1e7ce7cd89e62 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Fri, 10 Mar 2023 12:28:34 -0800 Subject: [PATCH 61/65] Address review comments Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/msg__struct.h.em | 10 +-- rosidl_generator_tests/CMakeLists.txt | 2 +- rosidl_generator_tests/package.xml | 1 + rosidl_generator_type_description/package.xml | 2 +- .../__init__.py | 2 + rosidl_runtime_c/src/type_hash.c | 72 +++++++++---------- 6 files changed, 46 insertions(+), 43 deletions(-) diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index 5f212b70a..0c882fd84 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -63,6 +63,11 @@ for member in message.structure.members: @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< +// Type Version Hash for interface +@{ hash_var = idl_structure_type_to_c_typename(message.structure.namespaced_type) + '__TYPE_VERSION_HASH' }@ +#define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['message'], line_final_backslash=True)) +static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; + // Constants defined in the message @[for constant in message.constants]@ @@ -178,11 +183,6 @@ typedef struct @(idl_structure_type_to_c_typename(message.structure.namespaced_t @(idl_declaration_to_c(member.type, member.name)); @[end for]@ } @(idl_structure_type_to_c_typename(message.structure.namespaced_type)); - -// Type Version Hash for interface -@{ hash_var = idl_structure_type_to_c_typename(message.structure.namespaced_type) + '__TYPE_VERSION_HASH' }@ -#define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['message'], line_final_backslash=True)) -static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; @#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/rosidl_generator_tests/CMakeLists.txt b/rosidl_generator_tests/CMakeLists.txt index a8746bc83..90784dabd 100644 --- a/rosidl_generator_tests/CMakeLists.txt +++ b/rosidl_generator_tests/CMakeLists.txt @@ -30,9 +30,9 @@ if(BUILD_TESTING) ament_lint_auto_find_test_dependencies() rosidl_generate_interfaces(${PROJECT_NAME} + ${test_interface_files_ACTION_FILES} ${test_interface_files_MSG_FILES} ${test_interface_files_SRV_FILES} - ${test_interface_files_ACTION_FILES} ADD_LINTER_TESTS SKIP_INSTALL ) diff --git a/rosidl_generator_tests/package.xml b/rosidl_generator_tests/package.xml index 9ee03be32..449bf8ef8 100644 --- a/rosidl_generator_tests/package.xml +++ b/rosidl_generator_tests/package.xml @@ -20,6 +20,7 @@ action_msgs ament_cmake_gtest + ament_cmake_pytest ament_lint_auto ament_lint_common ament_index_python diff --git a/rosidl_generator_type_description/package.xml b/rosidl_generator_type_description/package.xml index 88f87f978..4886a4fe7 100644 --- a/rosidl_generator_type_description/package.xml +++ b/rosidl_generator_type_description/package.xml @@ -2,7 +2,7 @@ rosidl_generator_type_description - 3.3.1 + 3.4.0 Generate hashes and descriptions of ROS 2 interface types, per REP-2011. Emerson Knapp diff --git a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py index fe3de28ca..cccc48699 100644 --- a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py +++ b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py @@ -258,6 +258,8 @@ def field_type_type_name(ftype: definition.AbstractType) -> str: isinstance(value_type, definition.NamedType) ): value_type_name = 'FIELD_TYPE_NESTED_TYPE' + else: + raise ValueError(f'Unknown value type {value_type}') return value_type_name + name_suffix diff --git a/rosidl_runtime_c/src/type_hash.c b/rosidl_runtime_c/src/type_hash.c index b641d96f0..ce13112ec 100644 --- a/rosidl_runtime_c/src/type_hash.c +++ b/rosidl_runtime_c/src/type_hash.c @@ -23,11 +23,7 @@ static const char RIHS01_PREFIX[] = "RIHS01_"; static const size_t RIHS_VERSION_IDX = 4; static const size_t RIHS_PREFIX_LEN = 7; static const size_t RIHS01_STRING_LEN = 71; // RIHS_PREFIX_LEN + (ROSIDL_TYPE_HASH_SIZE * 2); - -static bool _ishexdigit(char c) -{ - return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); -} +static const uint8_t INVALID_NIBBLE = 0xff; /// Translate a single character hex digit to a nibble static uint8_t _xatoi(char c) @@ -41,21 +37,7 @@ static uint8_t _xatoi(char c) if (c >= 'a' && c <= 'f') { return c - 'a' + 0xa; } - return -1; -} - -/// Tranlate a byte value to two hex characters -static void _xitoa(uint8_t val, char * dest) -{ - uint8_t nibble = 0; - for (size_t n = 0; n < 2; n++) { - nibble = (val >> (4 * (1 - n))) & 0x0f; - if (nibble < 0xa) { - dest[n] = '0' + nibble; - } else { // 0xa <= nibble < 0x10 - dest[n] = 'a' + (nibble - 0xa); - } - } + return INVALID_NIBBLE; } rosidl_type_hash_t @@ -86,8 +68,26 @@ rosidl_stringify_type_hash( } local_output[RIHS01_STRING_LEN] = '\0'; memcpy(local_output, RIHS01_PREFIX, RIHS_PREFIX_LEN); + + uint8_t nibble = 0; + char * dest = NULL; for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { - _xitoa(type_hash->value[i], local_output + RIHS_PREFIX_LEN + (i * 2)); + // Translate byte into two hex characters + dest = local_output + RIHS_PREFIX_LEN + (i * 2); + // First character is top half of byte + nibble = (type_hash->value[i] >> 4) & 0x0f; + if (nibble < 0xa) { + dest[0] = '0' + nibble; + } else { + dest[0] = 'a' + (nibble - 0xa); + } + // Second character is bottom half of byte + nibble = (type_hash->value[i] >> 0) & 0x0f; + if (nibble < 0xa) { + dest[1] = '0' + nibble; + } else { + dest[1] = 'a' + (nibble - 0xa); + } } *output_string = local_output; @@ -103,6 +103,8 @@ rosidl_parse_type_hash_string( RCUTILS_CHECK_ARGUMENT_FOR_NULL(hash_out, RCUTILS_RET_INVALID_ARGUMENT); hash_out->version = 0; size_t input_len = strlen(type_hash_string); + uint8_t hexbyte_top_nibble; + uint8_t hexbyte_bot_nibble; // Check prefix if (input_len < RIHS_PREFIX_LEN) { @@ -115,37 +117,35 @@ rosidl_parse_type_hash_string( } // Parse version - char version_top = type_hash_string[RIHS_VERSION_IDX]; - char version_bot = type_hash_string[RIHS_VERSION_IDX + 1]; - if (!(_ishexdigit(version_top) && _ishexdigit(version_bot))) { + hexbyte_top_nibble = _xatoi(type_hash_string[RIHS_VERSION_IDX]); + hexbyte_bot_nibble = _xatoi(type_hash_string[RIHS_VERSION_IDX + 1]); + if (hexbyte_top_nibble == INVALID_NIBBLE || hexbyte_bot_nibble == INVALID_NIBBLE) { RCUTILS_SET_ERROR_MSG("RIHS version is not a 2-digit hex string."); return RCUTILS_RET_INVALID_ARGUMENT; } - hash_out->version = (_xatoi(version_top) << 4) + _xatoi(version_bot); + hash_out->version = (hexbyte_top_nibble << 4) + hexbyte_bot_nibble; if (hash_out->version != 1) { RCUTILS_SET_ERROR_MSG("Do not know how to parse RIHS version."); return RCUTILS_RET_INVALID_ARGUMENT; } + + // Check total length if (input_len != RIHS01_STRING_LEN) { RCUTILS_SET_ERROR_MSG("RIHS string is the incorrect size to contain a RIHS01 value."); return RCUTILS_RET_INVALID_ARGUMENT; } + + // Parse hash value const char * value_str = type_hash_string + RIHS_PREFIX_LEN; - for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE * 2; i++) { - if (!_ishexdigit(value_str[i])) { - RCUTILS_SET_ERROR_MSG("Type hash string value contained non-hex-digit character."); - return RCUTILS_RET_INVALID_ARGUMENT; - } - } for (size_t i = 0; i < ROSIDL_TYPE_HASH_SIZE; i++) { - // No error checking on byte values because the whole string was checked in prior loop - uint8_t byte_val = (_xatoi(value_str[i * 2]) << 4) + _xatoi(value_str[(i * 2) + 1]); - if (byte_val < 0) { - RCUTILS_SET_ERROR_MSG("Type hash string value did not contain only hex digits."); + hexbyte_top_nibble = _xatoi(value_str[i * 2]); + hexbyte_bot_nibble = _xatoi(value_str[i * 2 + 1]); + if (hexbyte_top_nibble == INVALID_NIBBLE || hexbyte_bot_nibble == INVALID_NIBBLE) { + RCUTILS_SET_ERROR_MSG("Type hash string value contained non-hexdigit character."); return RCUTILS_RET_INVALID_ARGUMENT; } - hash_out->value[i] = (char)byte_val; + hash_out->value[i] = (hexbyte_top_nibble << 4) + hexbyte_bot_nibble; } return RCUTILS_RET_OK; } From 6fff5a6d0a66e1a958b842656264b93b0b36b7fa Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Fri, 10 Mar 2023 12:50:19 -0800 Subject: [PATCH 62/65] Add new test for nested types Signed-off-by: Emerson Knapp --- .../test/test_serializers.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/rosidl_generator_type_description/test/test_serializers.py b/rosidl_generator_type_description/test/test_serializers.py index d50f96b4d..868cde18a 100644 --- a/rosidl_generator_type_description/test/test_serializers.py +++ b/rosidl_generator_type_description/test/test_serializers.py @@ -13,6 +13,7 @@ # limitations under the License. from rosidl_generator_type_description import serialize_field_type +from rosidl_generator_type_description import serialize_individual_type_description from rosidl_parser import definition @@ -52,3 +53,39 @@ def test_field_type_serializer(): } result = serialize_field_type(test_type) assert result == expected + + +def test_nested_type_serializer(): + namespaced_type = definition.NamespacedType(['my_pkg', 'msg'], 'TestThing') + referenced_type = definition.NamespacedType(['other_pkg', 'msg'], 'RefThing') + nested_referenced_type = definition.UnboundedSequence(referenced_type) + members = [ + definition.Member(referenced_type, 'ref_thing'), + definition.Member(nested_referenced_type, 'ref_things') + ] + expected = { + 'type_name': 'my_pkg/msg/TestThing', + 'fields': [ + { + 'name': 'ref_thing', + 'type': { + 'type_id': 1, + 'capacity': 0, + 'string_capacity': 0, + 'nested_type_name': 'other_pkg/msg/RefThing', + }, + }, + { + 'name': 'ref_things', + 'type': { + 'type_id': 145, + 'capacity': 0, + 'string_capacity': 0, + 'nested_type_name': 'other_pkg/msg/RefThing', + }, + }, + ], + } + result = serialize_individual_type_description(namespaced_type, members) + + assert result == expected From d642a4860c0748f6e883b23b1e80d576e5b61255 Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Sun, 12 Mar 2023 23:22:06 -0700 Subject: [PATCH 63/65] Generate hashes in C for services of action Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index ecfd437fb..2d2f222c0 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -126,6 +126,10 @@ TEMPLATE( type_hash=type_hash['feedback']) }@ +@{ hash_var = idl_structure_type_to_c_typename(action.send_goal_service.namespaced_type) + '__TYPE_VERSION_HASH' }@ +#define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['send_goal_service']['service'], line_final_backslash=True)) +static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; + @{ TEMPLATE( 'msg__struct.h.em', @@ -150,6 +154,10 @@ TEMPLATE( type_hash=type_hash['send_goal_service']['event_message']) }@ +@{ hash_var = idl_structure_type_to_c_typename(action.get_result_service.namespaced_type) + '__TYPE_VERSION_HASH' }@ +#define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['get_result_service']['service'], line_final_backslash=True)) +static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; + @{ TEMPLATE( 'msg__struct.h.em', From cfd74df1ac492d8520bae7c9cafebc76a6fbe69d Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 13 Mar 2023 12:40:18 -0700 Subject: [PATCH 64/65] Rename type_description to type_description_msg Signed-off-by: Emerson Knapp --- .../resource/HashedTypeDescription.schema.json | 4 ++-- .../rosidl_generator_type_description/__init__.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json b/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json index 1563ff285..14f17a6c0 100644 --- a/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json +++ b/rosidl_generator_type_description/resource/HashedTypeDescription.schema.json @@ -13,9 +13,9 @@ { "$ref": "#/$defs/ActionHashes" } ] }, - "type_description": { "$ref": "#/$defs/TypeDescription" } + "type_description_msg": { "$ref": "#/$defs/TypeDescription" } }, - "required": ["hashes", "type_description"], + "required": ["hashes", "type_description_msg"], "additionalProperties": false, "$defs": { "MessageHash": { diff --git a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py index cccc48699..117a6c951 100644 --- a/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py +++ b/rosidl_generator_type_description/rosidl_generator_type_description/__init__.py @@ -432,9 +432,9 @@ def add_referenced_type(individual_type_description): with include_path.open('r') as include_file: include_json = json.load(include_file) - type_description = include_json['type_description'] - add_referenced_type(type_description['type_description']) - for rt in type_description['referenced_type_descriptions']: + type_description_msg = include_json['type_description_msg'] + add_referenced_type(type_description_msg['type_description']) + for rt in type_description_msg['referenced_type_descriptions']: add_referenced_type(rt) self.full_type_description = { @@ -445,7 +445,7 @@ def add_referenced_type(individual_type_description): hashed_type_description = { 'hashes': self._calculate_hash_tree(), - 'type_description': self.full_type_description, + 'type_description_msg': self.full_type_description, } json_path = output_dir / self.rel_path.with_suffix('.json') From a339dd9107d50530fc53730f7c3d11831e780bac Mon Sep 17 00:00:00 2001 From: Emerson Knapp Date: Mon, 13 Mar 2023 12:49:32 -0700 Subject: [PATCH 65/65] Add comment note about __INIT defines Signed-off-by: Emerson Knapp --- rosidl_generator_c/resource/idl__struct.h.em | 3 +++ rosidl_generator_c/resource/msg__struct.h.em | 1 + 2 files changed, 4 insertions(+) diff --git a/rosidl_generator_c/resource/idl__struct.h.em b/rosidl_generator_c/resource/idl__struct.h.em index 2d2f222c0..9d6b2d1c4 100644 --- a/rosidl_generator_c/resource/idl__struct.h.em +++ b/rosidl_generator_c/resource/idl__struct.h.em @@ -61,6 +61,7 @@ from rosidl_parser.definition import Service @[for service in content.get_elements_of_type(Service)]@ @{ hash_var = idl_structure_type_to_c_typename(service.namespaced_type) + '__TYPE_VERSION_HASH' }@ +// Note: this define is for MSVC, where the static const var can't be used in downstream aggregate initializers #define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['service'], line_final_backslash=True)) static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; @@ -127,6 +128,7 @@ TEMPLATE( }@ @{ hash_var = idl_structure_type_to_c_typename(action.send_goal_service.namespaced_type) + '__TYPE_VERSION_HASH' }@ +// Note: this define is for MSVC, where the static const var can't be used in downstream aggregate initializers #define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['send_goal_service']['service'], line_final_backslash=True)) static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; @@ -155,6 +157,7 @@ TEMPLATE( }@ @{ hash_var = idl_structure_type_to_c_typename(action.get_result_service.namespaced_type) + '__TYPE_VERSION_HASH' }@ +// Note: this define is for MSVC, where the static const var can't be used in downstream aggregate initializers #define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['get_result_service']['service'], line_final_backslash=True)) static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT; diff --git a/rosidl_generator_c/resource/msg__struct.h.em b/rosidl_generator_c/resource/msg__struct.h.em index 0c882fd84..e75077af9 100644 --- a/rosidl_generator_c/resource/msg__struct.h.em +++ b/rosidl_generator_c/resource/msg__struct.h.em @@ -65,6 +65,7 @@ for member in message.structure.members: @#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< // Type Version Hash for interface @{ hash_var = idl_structure_type_to_c_typename(message.structure.namespaced_type) + '__TYPE_VERSION_HASH' }@ +// Note: this define is for MSVC, where the static const var can't be used in downstream aggregate initializers #define @(hash_var)__INIT @(type_hash_to_c_definition(type_hash['message'], line_final_backslash=True)) static const rosidl_type_hash_t @(hash_var) = @(hash_var)__INIT;