diff --git a/ESSArch_Core/essxml/Generator/tests/test_generate.py b/ESSArch_Core/essxml/Generator/tests/test_generate.py index 775275d70..89b05802b 100644 --- a/ESSArch_Core/essxml/Generator/tests/test_generate.py +++ b/ESSArch_Core/essxml/Generator/tests/test_generate.py @@ -88,7 +88,8 @@ def test_generate_namespaces(self): "xsi:schemaLocation": "http://www.w3.org/1999/xlink schemas/xlink.xsd", } - self.generator.generate({self.fname: {'spec': specification, 'data': info}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': info}}): + pass tree = etree.parse(self.fname) root = tree.getroot() @@ -105,7 +106,8 @@ def test_generate_empty_element(self): specification = {'-name': "foo"} with self.assertRaises(AssertionError): - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.exists(self.fname)) @@ -122,7 +124,8 @@ def test_generate_empty_element_with_children(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) def test_generate_empty_element_with_empty_children(self): @@ -136,7 +139,8 @@ def test_generate_empty_element_with_empty_children(self): } with self.assertRaises(AssertionError): - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.exists(self.fname)) @@ -151,7 +155,8 @@ def test_generate_empty_element_with_empty_children_with_allow_empty(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) def test_generate_empty_element_with_empty_attribute(self): @@ -168,7 +173,8 @@ def test_generate_empty_element_with_empty_attribute(self): } with self.assertRaises(AssertionError): - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.exists(self.fname)) @@ -184,7 +190,8 @@ def test_generate_empty_element_with_empty_attribute_with_allow_empty_on_attribu ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) root = tree.getroot() self.assertEqual(len(root.xpath('//foo[@bar=""]')), 1) @@ -205,13 +212,15 @@ def test_generate_multiple_element_same_name_same_level(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual(len(tree.findall('.//bar')), 2) def test_generate_empty_element_with_allowEmpty(self): specification = {'-name': "foo", "-allowEmpty": 1} - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -238,7 +247,8 @@ def test_generate_empty_element_with_hideEmptyContent(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -264,7 +274,8 @@ def test_generate_empty_element_without_hideEmptyContent(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -295,7 +306,8 @@ def test_generate_element_with_content_and_hideEmptyContent(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -321,7 +333,8 @@ def test_generate_element_with_empty_child_and_hideEmptyContent(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -353,7 +366,8 @@ def test_generate_element_with_foreach(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': data}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': data}}): + pass tree = etree.parse(self.fname) root = tree.getroot() @@ -382,7 +396,8 @@ def test_generate_element_with_foreach_dict(self): ]) } - self.generator.generate({self.fname: {'spec': specification, 'data': data}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': data}}): + pass tree = etree.parse(self.fname) root = tree.getroot() @@ -414,7 +429,8 @@ def test_generate_element_replace_existing(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) root = tree.getroot() @@ -455,7 +471,8 @@ def test_generate_element_replace_existing_multiple_attributes(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) root = tree.getroot() @@ -524,7 +541,8 @@ def test_generate_element_replace_existing_index(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) root = tree.getroot() elements = root.xpath('//bar') @@ -573,7 +591,8 @@ def test_generate_element_replace_existing_with_foreach(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': data}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': data}}): + pass tree = etree.parse(self.fname) root = tree.getroot() elements = root.xpath('//foo') @@ -616,7 +635,8 @@ def test_generate_element_ignore_existing_with_foreach(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': data}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': data}}): + pass tree = etree.parse(self.fname) root = tree.getroot() @@ -645,7 +665,8 @@ def test_generate_element_with_empty_child_with_containsFiles_and_hideEmptyConte ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -681,7 +702,8 @@ def test_generate_element_with_empty_child_with_containsFiles_and_hideEmptyConte ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -711,7 +733,8 @@ def test_generate_element_with_empty_child_with_containsFiles_and_files_and_hide ] } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass self.assertTrue(os.path.exists(self.fname)) tree = etree.parse(self.fname) @@ -736,7 +759,8 @@ def test_generate_element_with_content(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual("bar", etree.tostring(tree.getroot(), encoding='unicode')) @@ -749,7 +773,8 @@ def test_generate_required_element_with_content(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual("bar", etree.tostring(tree.getroot(), encoding='unicode')) @@ -760,7 +785,8 @@ def test_generate_empty_required_element(self): } with self.assertRaises(ValueError): - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.exists(self.fname)) @@ -794,8 +820,8 @@ def test_generate_nested_empty_required_element(self): with self.assertRaisesRegex(ValueError, re.escape( "Missing value for required element '/foo[0]/bar[1]/baz[2]'")): - - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.exists(self.fname)) @@ -814,7 +840,8 @@ def test_generate_empty_element_with_single_attribute(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual('', etree.tostring(tree.getroot(), encoding='unicode')) @@ -825,7 +852,8 @@ def test_generate_element_with_empty_attribute(self): '#content': [{'text': 'baz'}], } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual('baz', etree.tostring(tree.getroot(), encoding='unicode')) @@ -837,7 +865,8 @@ def test_generate_element_with_nameless_attribute(self): } with self.assertRaises(ValueError): - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.isfile(self.fname)) @@ -864,7 +893,8 @@ def test_generate_empty_element_with_multiple_attribute(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual( '', @@ -887,7 +917,8 @@ def test_generate_required_attribute_with_content(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual('', etree.tostring(tree.getroot(), encoding='unicode')) @@ -903,7 +934,8 @@ def test_generate_empty_required_attribute(self): } with self.assertRaises(ValueError): - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.exists(self.fname)) @@ -943,7 +975,8 @@ def test_generate_nested_empty_required_attribute(self): with self.assertRaisesRegex(ValueError, re.escape( "Missing value for required attribute 'test' on element '/foo[0]/bar[1]/baz[2]'")): - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertFalse(os.path.exists(self.fname)) @@ -963,7 +996,8 @@ def test_generate_element_with_content_and_attribute(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual( 'bar', @@ -985,7 +1019,8 @@ def test_generate_empty_element_with_attribute_using_var(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}): + pass tree = etree.parse(self.fname) self.assertEqual('', etree.tostring(tree.getroot(), encoding='unicode')) @@ -999,7 +1034,8 @@ def test_generate_element_with_content_using_var(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}): + pass tree = etree.parse(self.fname) self.assertEqual('baz', etree.tostring(tree.getroot(), encoding='unicode')) @@ -1009,7 +1045,8 @@ def test_generate_element_with_nested_xml_content(self): '-nestedXMLContent': 'bar' } - self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}): + pass tree = etree.parse(self.fname) self.assertEqual('\n baz\n', etree.tostring(tree.getroot(), encoding='unicode')) @@ -1034,7 +1071,8 @@ def test_generate_element_with_requiredParameters_and_required_var(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}): + pass tree = etree.parse(self.fname) self.assertEqual( '\n \n baz\n \n', @@ -1063,7 +1101,8 @@ def test_generate_element_with_requiredParameters_and_no_required_var(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': {'foo': 'baz'}}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'foo': 'baz'}}}): + pass tree = etree.parse(self.fname) self.assertEqual('', etree.tostring(tree.getroot(), encoding='unicode')) @@ -1082,7 +1121,8 @@ def test_generate_element_with_children(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'bar': 'baz'}}}): + pass tree = etree.parse(self.fname) self.assertEqual( @@ -1113,7 +1153,8 @@ def test_skipIfNoChildren_with_empty_child(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual( @@ -1138,7 +1179,8 @@ def test_skipIfNoChildren_with_non_empty_child(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual( @@ -1164,7 +1206,8 @@ def test_hide_content_if_missing_with_missing(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertEqual( @@ -1190,7 +1233,8 @@ def test_hide_content_if_missing_with_not_missing(self): ] } - self.generator.generate({self.fname: {'spec': specification, 'data': {'baz': 'value'}}}) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'baz': 'value'}}}): + pass tree = etree.parse(self.fname) self.assertEqual( @@ -1234,7 +1278,8 @@ def test_element_with_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) num_of_files = 0 @@ -1279,12 +1324,13 @@ def test_multiple_to_create_with_reference_from_second_to_first(self): first_fname = normalize_path(os.path.join(self.xmldir, 'nested', "first.xml")) second_fname = normalize_path(os.path.join(self.xmldir, 'nested', "second.xml")) - self.generator.generate( + for _ in self.generator.generate( OrderedDict([ (first_fname, {'spec': specification}), (second_fname, {'spec': specification}) ]) - ) + ): + pass tree1 = etree.parse(first_fname) tree2 = etree.parse(second_fname) @@ -1298,12 +1344,13 @@ def test_multiple_to_create_with_reference_from_second_to_first(self): self.assertEqual(bars2[0].text, first_fname) # again, but with relpath set - self.generator.generate( + for _ in self.generator.generate( OrderedDict([ (first_fname, {'spec': specification}), (second_fname, {'spec': specification}) ]), relpath=self.xmldir - ) + ): + pass tree1 = etree.parse(first_fname) tree2 = etree.parse(second_fname) @@ -1354,12 +1401,13 @@ def test_multiple_to_create_with_files(self): extra_fname = os.path.join(self.xmldir, "extra.xml") - self.generator.generate( + for _ in self.generator.generate( OrderedDict([ (self.fname, {'spec': specification}), (extra_fname, {'spec': specification}) ]), folderToParse=self.datadir - ) + ): + pass tree1 = etree.parse(self.fname) tree2 = etree.parse(extra_fname) @@ -1391,7 +1439,8 @@ def test_element_with_containsFiles_without_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass self.assertTrue(os.path.exists(self.fname)) def test_element_with_files_and_namespace(self): @@ -1437,7 +1486,8 @@ def test_element_with_files_and_namespace(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) num_of_files = 0 @@ -1497,7 +1547,8 @@ def test_element_with_filtered_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) num_of_files = 0 @@ -1535,7 +1586,8 @@ def test_position_alphabetically(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) root = tree.getroot() a = root.find('.//a') @@ -1558,7 +1610,8 @@ def test_position_non_alphabetically(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) root = tree.getroot() a = root.find('.//a') @@ -1585,7 +1638,8 @@ def test_position_with_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) root = tree.getroot() a = root.find('.//a') @@ -1607,7 +1661,8 @@ def test_position_with_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) root = tree.getroot() @@ -1630,7 +1685,8 @@ def test_position_with_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) root = tree.getroot() a = root.find('.//a') @@ -1652,7 +1708,8 @@ def test_position_with_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) root = tree.getroot() a = root.find('.//a') @@ -1694,7 +1751,8 @@ def test_position_with_files(self): ], } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass tree = etree.parse(self.fname) root = tree.getroot() a = root.find('.//a') @@ -1719,7 +1777,8 @@ def test_insert_from_xml_string(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass append_xml = 'appended text' target = self.generator.find_element('foo') @@ -1747,7 +1806,8 @@ def test_insert_from_xml_file(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass append_xml = os.path.join(self.xmldir, 'append.xml') append_xml_string = 'appended text' @@ -1785,7 +1845,8 @@ def test_insert_element_with_namespace(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -1828,7 +1889,8 @@ def test_insert_element_at_index(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -1854,7 +1916,8 @@ def test_insert_element_at_index(self): self.assertLess(root.index(appended), root.index(foo)) - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -1911,7 +1974,8 @@ def test_insert_element_before_element(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -1946,7 +2010,8 @@ def test_insert_element_before_non_existing_element(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -1995,7 +2060,8 @@ def test_insert_element_after_element(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -2030,7 +2096,8 @@ def test_insert_element_after_non_existing_element(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -2058,7 +2125,8 @@ def test_insert_element_before_and_after_element(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -2095,7 +2163,8 @@ def test_insert_nested_elements_with_namespace(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -2145,7 +2214,8 @@ def test_insert_element_with_content(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -2182,7 +2252,8 @@ def test_insert_element_with_attribute(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -2223,7 +2294,8 @@ def test_insert_empty_element(self): ] } - self.generator.generate({self.fname: {'spec': specification}}) + for _ in self.generator.generate({self.fname: {'spec': specification}}): + pass tree = etree.parse(self.fname) self.assertIsNone(tree.find('.//appended')) @@ -2285,7 +2357,8 @@ def test_external(self): }, } - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass self.assertTrue(os.path.isfile(self.fname)) external1_path = os.path.join(self.external1, 'external.xml') @@ -2349,7 +2422,8 @@ def test_external_with_files(self): with open(os.path.join(self.external2, "file1.pdf"), "w") as f: f.write('a pdf file') - self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir) + for _ in self.generator.generate({self.fname: {'spec': specification}}, folderToParse=self.datadir): + pass self.assertTrue(os.path.isfile(self.fname)) tree = etree.parse(self.fname) @@ -2398,10 +2472,10 @@ def test_external_info(self): }, } - self.generator.generate( - {self.fname: {'spec': specification, 'data': {'foo': 'bar'}}}, - folderToParse=self.datadir - ) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'foo': 'bar'}}}, + folderToParse=self.datadir): + pass + self.assertTrue(os.path.isfile(self.fname)) external1_path = os.path.join(self.external1, 'external.xml') @@ -2446,10 +2520,9 @@ def test_external_nsmap(self): } }, } - self.generator.generate( - {self.fname: {'spec': specification, 'data': {'foo': 'bar'}}}, - folderToParse=self.datadir - ) + for _ in self.generator.generate({self.fname: {'spec': specification, 'data': {'foo': 'bar'}}}, + folderToParse=self.datadir): + pass external1_path = os.path.join(self.external1, 'external.xml') external2_path = os.path.join(self.external2, 'external.xml') @@ -2498,7 +2571,7 @@ def test_external_nsmap_collision(self): } }, } - self.generator.generate( + for _ in self.generator.generate( { self.fname: { 'spec': specification, @@ -2506,7 +2579,8 @@ def test_external_nsmap_collision(self): } }, folderToParse=self.datadir - ) + ): + pass external1_path = os.path.join(self.external1, 'external.xml') external2_path = os.path.join(self.external2, 'external.xml') diff --git a/ESSArch_Core/essxml/Generator/xmlGenerator.py b/ESSArch_Core/essxml/Generator/xmlGenerator.py index 7b1bf2449..edf731d4e 100644 --- a/ESSArch_Core/essxml/Generator/xmlGenerator.py +++ b/ESSArch_Core/essxml/Generator/xmlGenerator.py @@ -476,7 +476,6 @@ def parse(self, info, nsmap=None): def find_files_in_path_not_in_external_dirs(fid, path, external, algorithm, rootdir=""): - files = [] for root, dirnames, filenames in walk(path): dirnames[:] = [d for d in dirnames if d not in [e[1] for e in external]] @@ -484,23 +483,18 @@ def find_files_in_path_not_in_external_dirs(fid, path, external, algorithm, root filepath = os.path.join(root, fname) relpath = os.path.relpath(filepath, path) - fileinfo = parse_file(filepath, fid, relpath, algorithm=algorithm, rootdir=rootdir) - files.append(fileinfo) - return files + yield parse_file(filepath, fid, relpath, algorithm=algorithm, rootdir=rootdir) def parse_files(fid, path, external, algorithm, rootdir): - files = [] if os.path.isfile(path): relpath = os.path.basename(path) - file_info = parse_file(path, fid, relpath, algorithm=algorithm) - files.append(file_info) + yield parse_file(path, fid, relpath, algorithm=algorithm) elif os.path.isdir(path): - found_files = find_files_in_path_not_in_external_dirs(fid, path, external, algorithm, rootdir) - files.extend(found_files) - return files + for e in find_files_in_path_not_in_external_dirs(fid, path, external, algorithm, rootdir): + yield e class XMLGenerator(object): @@ -581,17 +575,22 @@ def generate(self, filesToCreate, folderToParse=None, extra_paths_to_parse=None, external_to_create = { os.path.join(folderToParse, ptr_file_path): {'spec': ext_spec, 'data': ext_info} } - external_gen.generate(external_to_create, os.path.join(folderToParse, ext_dir, sub_dir)) + for _ in external_gen.generate(external_to_create, os.path.join(folderToParse, ext_dir, sub_dir)): + yield filepath = os.path.join(folderToParse, ptr_file_path) fileinfo = parse_file(filepath, self.fid, ptr_file_path, algorithm=algorithm, rootdir=sub_dir) files.append(fileinfo) - files.extend(parse_files(self.fid, folderToParse, external, algorithm, rootdir="")) + for e in parse_files(self.fid, folderToParse, external, algorithm, rootdir=""): + files.extend(e) if isinstance(e, list) else files.append(e) + yield for path in extra_paths_to_parse: - files.extend(parse_files(self.fid, path, external, algorithm, rootdir=path)) + for e in parse_files(self.fid, path, external, algorithm, rootdir=path): + files.extend(e) if isinstance(e, list) else files.append(e) + yield for idx, f in enumerate(self.toCreate): fname = f['file'] diff --git a/ESSArch_Core/fixity/receipt/backends/xml.py b/ESSArch_Core/fixity/receipt/backends/xml.py index 6f2909afe..ebffb75e6 100644 --- a/ESSArch_Core/fixity/receipt/backends/xml.py +++ b/ESSArch_Core/fixity/receipt/backends/xml.py @@ -54,5 +54,6 @@ def create(self, template, destination, outcome, short_message, message, date=No data[u'ärenden'].append(a_data) files_to_create = {destination: {'spec': spec, 'data': data}} - XMLGenerator().generate(files_to_create) + for _ in XMLGenerator().generate(files_to_create): + pass logger.info(u'XML receipt created: {}'.format(destination)) diff --git a/ESSArch_Core/fixity/validation/tests/tests.py b/ESSArch_Core/fixity/validation/tests/tests.py index 59781723d..8ee19fd39 100644 --- a/ESSArch_Core/fixity/validation/tests/tests.py +++ b/ESSArch_Core/fixity/validation/tests/tests.py @@ -505,7 +505,8 @@ def create_files(self): return files def generate_xml(self): - self.generator.generate(self.filesToCreate, folderToParse=self.datadir) + for _ in self.generator.generate(self.filesToCreate, folderToParse=self.datadir): + pass def test_validation_without_files(self): root = etree.fromstring('') @@ -985,10 +986,12 @@ def create_files(self): return files def generate_mets_xml(self): - self.generator.generate(self.mets_spec, folderToParse=self.datadir) + for _ in self.generator.generate(self.mets_spec, folderToParse=self.datadir): + pass def generate_premis_xml(self): - self.generator.generate(self.premis_spec, folderToParse=self.datadir) + for _ in self.generator.generate(self.premis_spec, folderToParse=self.datadir): + pass def test_validation_without_files(self): root = etree.fromstring('') diff --git a/ESSArch_Core/ip/tasks.py b/ESSArch_Core/ip/tasks.py index e5aeacc45..8357dadb1 100644 --- a/ESSArch_Core/ip/tasks.py +++ b/ESSArch_Core/ip/tasks.py @@ -29,7 +29,7 @@ get_premis_ip_object_element_spec, normalize_path, zip_directory, -) + get_tree_size_and_count) User = get_user_model() @@ -38,7 +38,12 @@ class GenerateContentMets(DBTask): event_type = 50600 def run(self): - generate_content_mets(self.get_information_package()) + progress = 0 + _, count = get_tree_size_and_count(self.get_information_package().object_path) + self.set_progress(progress, total=count) + for _ in generate_content_mets(self.get_information_package()): + progress += 1 + self.set_progress(progress, total=count) def event_outcome_success(self): ip = self.get_information_package() @@ -49,7 +54,12 @@ class GeneratePackageMets(DBTask): event_type = 50600 def run(self): - generate_package_mets(self.get_information_package()) + progress = 0 + _, count = get_tree_size_and_count(self.get_information_package().object_path) + self.set_progress(progress, total=count) + for _ in generate_package_mets(self.get_information_package()): + progress += 1 + self.set_progress(progress, total=count) def event_outcome_success(self): ip = self.get_information_package() @@ -60,7 +70,12 @@ class GeneratePremis(DBTask): event_type = 50600 def run(self): - generate_premis(self.get_information_package()) + progress = 0 + _, count = get_tree_size_and_count(self.get_information_package().object_path) + self.set_progress(progress, total=count) + for _ in generate_premis(self.get_information_package()): + progress += 1 + self.set_progress(progress, total=count) def event_outcome_success(self): ip = self.get_information_package() @@ -71,7 +86,12 @@ class GenerateEventsXML(DBTask): event_type = 50600 def run(self): - generate_events_xml(self.get_information_package()) + progress = 0 + _, count = get_tree_size_and_count(self.get_information_package().object_path) + self.set_progress(progress, total=count) + for _ in generate_events_xml(self.get_information_package()): + progress += 1 + self.set_progress(progress, total=count) def event_outcome_success(self): ip = self.get_information_package() diff --git a/ESSArch_Core/ip/utils.py b/ESSArch_Core/ip/utils.py index a116de133..841d80e47 100644 --- a/ESSArch_Core/ip/utils.py +++ b/ESSArch_Core/ip/utils.py @@ -79,7 +79,7 @@ def generate_content_mets(ip): algorithm = ip.get_checksum_algorithm() generator = XMLGenerator() - generator.generate(files_to_create, folderToParse=ip.object_path, algorithm=algorithm) + yield from generator.generate(files_to_create, folderToParse=ip.object_path, algorithm=algorithm) ip.content_mets_path = mets_path ip.content_mets_create_date = timestamp_to_datetime(creation_date(mets_path)).isoformat() @@ -115,7 +115,7 @@ def generate_package_mets(ip): algorithm = ip.get_checksum_algorithm() generator = XMLGenerator() - generator.generate(files_to_create, folderToParse=ip.object_path, algorithm=algorithm) + yield from generator.generate(files_to_create, folderToParse=ip.object_path, algorithm=algorithm) ip.package_mets_path = normalize_path(xmlpath) ip.package_mets_create_date = timestamp_to_datetime(creation_date(xmlpath)).isoformat() @@ -137,7 +137,7 @@ def generate_premis(ip): } algorithm = ip.get_checksum_algorithm() generator = XMLGenerator() - generator.generate(files_to_create, folderToParse=ip.object_path, algorithm=algorithm) + yield from generator.generate(files_to_create, folderToParse=ip.object_path, algorithm=algorithm) def generate_events_xml(ip): @@ -150,7 +150,7 @@ def generate_events_xml(ip): } algorithm = ip.get_checksum_algorithm() generator = XMLGenerator() - generator.generate(files_to_create, algorithm=algorithm) + yield from generator.generate(files_to_create, algorithm=algorithm) def download_schemas(ip, logger, verify): diff --git a/ESSArch_Core/tasks.py b/ESSArch_Core/tasks.py index 84302b873..a080ebcf7 100644 --- a/ESSArch_Core/tasks.py +++ b/ESSArch_Core/tasks.py @@ -121,11 +121,15 @@ def run(self, filesToCreate=None, folderToParse=None, extra_paths_to_parse=None, for _, v in filesToCreate.items(): v['data'] = fill_specification_data(v['data'], ip=ip, sa=sa) + progress = 0 + count = len(filesToCreate) + self.set_progress(progress, total=count) generator = XMLGenerator() - generator.generate( - filesToCreate, folderToParse=folderToParse, extra_paths_to_parse=extra_paths_to_parse, - parsed_files=parsed_files, algorithm=algorithm, - ) + for _ in generator.generate(filesToCreate, folderToParse=folderToParse, + extra_paths_to_parse=extra_paths_to_parse, + parsed_files=parsed_files, algorithm=algorithm): + progress += 1 + self.set_progress(progress, total=count) def undo(self, filesToCreate=None, folderToParse=None, extra_paths_to_parse=None, parsed_files=None, algorithm='SHA-256'):