diff --git a/Makefile b/Makefile index 9da7d2bf..f5fac61b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ .PHONY: all .PHONY: all_html all: artifacts/requirements.pdf artifacts/req-graph1.png \ - artifacts/req-graph2.png all_html + artifacts/requirements.xlsx artifacts/req-graph2.png \ + all_html # Adding new files (especially requirements) can not automatically # handled. The 'force' target tries to handle this. @@ -57,6 +58,7 @@ clean: artifacts/requirements.aux artifacts/requirements.dvi \ artifacts/requirements.log artifacts/requirements.out \ artifacts/requirements.pdf artifacts/requirements.toc \ + artifacts/requirements.xlsx \ add_data.py* rm -fr debian/rmtoo build diff --git a/contrib/template_project/Config.json b/contrib/template_project/Config.json index 21d7f307..8aca3697 100644 --- a/contrib/template_project/Config.json +++ b/contrib/template_project/Config.json @@ -3,29 +3,29 @@ "analytics": { "stop_on_errors": false } - }, + }, "requirements": { "input": { "commit_interval": { - "begin": "FILES", + "begin": "FILES", "end": "FILES" - }, - "default_language": "en_GB", + }, + "default_language": "en_GB", "dependency_notation": [ "Solved by" - ], + ], "directory": "requirements" - }, + }, "inventors": [ "flonatel" - ], + ], "stakeholders": [ - "development", - "management", - "users", + "development", + "management", + "users", "customers" ] - }, + }, "topics": { "ts_common": { "sources": [ @@ -36,54 +36,61 @@ "topic_root_node": "ReqsDocument", "constraints_dirs": [ "constraints" ], "testcases_dirs": [ "testcases" ] } - ] + ] ], "output": { "graph": [ { "output_filename": "artifacts/req-graph1.dot" } - ], + ], "graph2": [ { "output_filename": "artifacts/req-graph2.dot" } - ], + ], "html": [ { - "footer": "html/footer.html", - "header": "html/header.html", + "footer": "html/footer.html", + "header": "html/header.html", "output_directory": "artifacts/html" } - ], + ], "LatexJinja2": [ { "template_path": "latex/LatexJinja2", "output_filename": "artifacts/reqtopics.tex" } - ], + ], + "xls": [ + { + "output_filename": "artifacts/specification.xlsx", + "template_filename": "import/specification-template.xlsx", + "req_sheet": "Specification" + } + ], "prios": [ { "output_filename": "artifacts/reqsprios.tex" } - ], + ], "stats_burndown1": [ { - "output_filename": "artifacts/stats_burndown.csv", + "output_filename": "artifacts/stats_burndown.csv", "start_date": "2011-05-01" } - ], + ], "stats_reqs_cnt": [ { "output_filename": "artifacts/stats_reqs_cnt.csv" } - ], + ], "stats_sprint_burndown1": [ { - "output_filename": "artifacts/stats_sprint_burndown.csv", + "output_filename": "artifacts/stats_sprint_burndown.csv", "start_date": "2011-05-01" } - ], + ], "version1": [ { "output_filename": "artifacts/reqs-version.txt" diff --git a/contrib/template_project/import/specification-template.xlsx b/contrib/template_project/import/specification-template.xlsx new file mode 100755 index 00000000..904d9b7c Binary files /dev/null and b/contrib/template_project/import/specification-template.xlsx differ diff --git a/contrib/template_project/requirements/project.req b/contrib/template_project/requirements/project.req index 98af3f51..0f06b6bb 100644 --- a/contrib/template_project/requirements/project.req +++ b/contrib/template_project/requirements/project.req @@ -1,5 +1,5 @@ Name: project -Type: master requirement +Type: requirement Invented on: 2010-10-09 Invented by: flonatel Owner: development diff --git a/doc/Config1.json b/doc/Config1.json index 3b5d5d9a..18446faa 100644 --- a/doc/Config1.json +++ b/doc/Config1.json @@ -20,6 +20,11 @@ "stop_on_errors": false } }, + "import": { + "xls": { + "import_filename": "doc/import/requirements.xlsx" + } + }, "requirements": { "input": { "default_language": "en_GB", @@ -28,7 +33,8 @@ ] }, "inventors": [ - "flonatel" + "flonatel", + "kno" ], "stakeholders": [ "development", @@ -90,6 +96,11 @@ "output_filename": "artifacts/reqtopics.tex" } ], + "xls": [ + { + "output_filename": "artifacts/requirements.xlsx" + } + ], "oopricing1": [ { "output_filename": "artifacts/reqspricing" diff --git a/doc/requirements/Import.req b/doc/requirements/Import.req new file mode 100644 index 00000000..19486c52 --- /dev/null +++ b/doc/requirements/Import.req @@ -0,0 +1,12 @@ +Name: Import Requirements +Type: requirement +Invented on: 2018-07-01 +Invented by: kno +Description: Import requirements from other sources. +Note: Provide a simpler interface to the text files. +Owner: development +Status: not done +Priority: development:10 +Effort estimation: 5 +Topic: Import +Solved by: Import/Xls diff --git a/doc/requirements/Import/Xls.req b/doc/requirements/Import/Xls.req new file mode 100644 index 00000000..eda3d6b8 --- /dev/null +++ b/doc/requirements/Import/Xls.req @@ -0,0 +1,15 @@ +Name: Xlsx Import +Type: design decision +Invented on: 2018-09-23 +Invented by: kno +Owner: development +Description: Import requirement files from the exported xlsx + file. +Rationale: Some people prefer to handle their requirements in + familiar tools. +Status: not done +Priority: development:10 +Effort estimation: 3 +Topic: Import +Solved by: Import/XlsDefaultSolvedby Import/XlsFutureInventedOn + Import/XlsBackupImportFile diff --git a/doc/requirements/Import/XlsBackupImportFile.req b/doc/requirements/Import/XlsBackupImportFile.req new file mode 100644 index 00000000..66cf45c0 --- /dev/null +++ b/doc/requirements/Import/XlsBackupImportFile.req @@ -0,0 +1,12 @@ +Name: Backup Imported File +Type: requirement +Invented on: 2018-10-12 +Invented by: kno +Owner: development +Description: Xlsx files that have been successfully imported are + renamed to its current name with the current timestamp appended. +Rationale: Avoid importing files multiple times. +Status: assigned:kno:2018-10-12 +Priority: development:10 +Effort estimation: 1 +Topic: Import diff --git a/doc/requirements/Import/XlsDefaultSolvedby.req b/doc/requirements/Import/XlsDefaultSolvedby.req new file mode 100644 index 00000000..c2ef5875 --- /dev/null +++ b/doc/requirements/Import/XlsDefaultSolvedby.req @@ -0,0 +1,14 @@ +Name: Default SolvedBy argument +Type: requirement +Invented on: 2018-09-23 +Invented by: kno +Owner: development +Description: New requirements without a parent-requirement, i.e. it's + not in any \emph{SolvedBy} list, are attached to the first + requirement in the list. +Rationale: To make handling easier, don't burden people with hierarchy + and deeper complexities of the tool if it isn't required. +Status: finished:kno:2018-10-12:2h +Priority: development:10 +Effort estimation: 1 +Topic: Import diff --git a/doc/requirements/Import/XlsFutureInventedOn.req b/doc/requirements/Import/XlsFutureInventedOn.req new file mode 100644 index 00000000..27b06684 --- /dev/null +++ b/doc/requirements/Import/XlsFutureInventedOn.req @@ -0,0 +1,12 @@ +Name: Constrain valid Invented On +Type: requirement +Invented on: 2018-10-02 +Invented by: kno +Owner: development +Description: Imported requirements may not have a \emph{InventedOn} + date in the future. +Rationale: Leads to cryptic errors messages +Status: finished:kno:2018-10-06:4h +Priority: development:1 +Effort estimation: 1 +Topic: Import diff --git a/doc/requirements/OutputTextDocument.req b/doc/requirements/OutputTextDocument.req index 9982493f..2f112fa0 100644 --- a/doc/requirements/OutputTextDocument.req +++ b/doc/requirements/OutputTextDocument.req @@ -12,5 +12,5 @@ Priority: customers:8 Effort estimation: 8 Topic: Output Solved by: OutputBaseReqRefs OutputBaseTags OutputConfgbl OutputHTML - OutputPDF OutputTextSameBase Output/DiffOfTwoVersions + OutputPDF OutputXLS OutputTextSameBase Output/DiffOfTwoVersions # Added by rmtoo-normalize-dependencies diff --git a/doc/requirements/OutputXLS.req b/doc/requirements/OutputXLS.req new file mode 100644 index 00000000..8e836348 --- /dev/null +++ b/doc/requirements/OutputXLS.req @@ -0,0 +1,11 @@ +Name: Output of XLS +Topic: Output +Description: \textsl{rmtoo} \textbf{must} support genration of XLS file output with all requirements. +Status: finished +Owner: development +Invented by: kno +Invented on: 2010-02-12 +Rationale: Suits love to handle their data in Excel. +Type: design decision +Priority: management:8 +Effort estimation: 8 diff --git a/doc/requirements/Processing.req b/doc/requirements/Processing.req index 403ea353..4728d17e 100644 --- a/doc/requirements/Processing.req +++ b/doc/requirements/Processing.req @@ -14,5 +14,5 @@ Status: not done Priority: development:10 Effort estimation: 21 Topic: Basics -Solved by: Analytics AutomaticGeneration Input Output +Solved by: Analytics AutomaticGeneration Input Output Import # Added by rmtoo-normalize-dependencies diff --git a/doc/topics/Import.tic b/doc/topics/Import.tic new file mode 100644 index 00000000..06e7eacd --- /dev/null +++ b/doc/topics/Import.tic @@ -0,0 +1,2 @@ +Name: Import +IncludeRequirements: full diff --git a/doc/topics/ReqsDocument.tic b/doc/topics/ReqsDocument.tic index d6eb5320..1b5db28b 100644 --- a/doc/topics/ReqsDocument.tic +++ b/doc/topics/ReqsDocument.tic @@ -10,6 +10,7 @@ SubTopic: ReqTags SubTopic: Topic SubTopic: Input SubTopic: Output +SubTopic: Import SubTopic: EmacsMode SubTopic: Documentation SubTopic: AutoCreationOfArtifacts diff --git a/requirements.txt b/requirements.txt index e1ded690..756ee280 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ gitpython>=3.1.41 jinja2>=2.10 numpy odfpy==1.3.4 +six>=1.17 +openpyxl>=2.5.6 pylint pytest pytest-cov diff --git a/rmtoo/imports/__init__.py b/rmtoo/imports/__init__.py new file mode 100644 index 00000000..c04bf726 --- /dev/null +++ b/rmtoo/imports/__init__.py @@ -0,0 +1,12 @@ +"""rmtoo + + Free and Open Source Requirements Management Tool + + Imports __init__.py + +(c) 2019, 2020 by Kristoffer +SPDX-License-Identifier: GPL-3.0-or-later +This file is part of rmtoo. +For licensing details see COPYING + +""" diff --git a/rmtoo/imports/abcimports.py b/rmtoo/imports/abcimports.py new file mode 100644 index 00000000..8f52d2e7 --- /dev/null +++ b/rmtoo/imports/abcimports.py @@ -0,0 +1,22 @@ +'''Import Abstract Base Class + +''' +import abc +import sys + +if sys.version_info >= (3, 4): + ABC = abc.ABC +else: + ABC = abc.ABCMeta('ABC', (object,), {}) # compatible with Python 2 *and* 3 + + +class AbcImports(ABC): + '''Define an ABC for all imports classes''' + + @abc.abstractmethod + def __init__(self, self_cfg, import_dest): + raise NotImplementedError + + @abc.abstractmethod + def run(self): + raise NotImplementedError diff --git a/rmtoo/imports/xls.py b/rmtoo/imports/xls.py new file mode 100644 index 00000000..888ab0d6 --- /dev/null +++ b/rmtoo/imports/xls.py @@ -0,0 +1,165 @@ +"""Import xls spreadsheets again + +Suits love Excel. + +(c) 2020 by Kristoffer +SPDX-License-Identifier: GPL-3.0-or-later +This file is part of rmtoo. +For licensing details see COPYING + +""" +from __future__ import unicode_literals +import os +import codecs +import datetime +import distutils.file_util +from collections import OrderedDict +import openpyxl + +from rmtoo.lib.logging import tracer +from rmtoo.lib.RMTException import RMTException +from rmtoo.lib.configuration.Cfg import Cfg +from rmtoo.imports.abcimports import AbcImports + + +class XlsImport(AbcImports): + """Import an xls-sheet created in the output plugins""" + + default_config = { + "import_filename": None, + "requirement_ws": "Requirements", + "topics_sheet": "Topics", + } + + def __init__(self, self_cfg, import_dest): + tracer.info("called") + self.useable = False + self._cfg = dict(self.default_config) + self._cfg.update(self_cfg) + self._dest = dict() + self._entries = None + self._topics = None + + import_dest_cfg = Cfg(import_dest) + try: + req_dirs = import_dest_cfg.get_rvalue("requirements_dirs") + if req_dirs[0] and os.path.isdir(req_dirs[0]): + self.useable = True + self._dest["requirements_dirs"] = req_dirs[0] + except RMTException: + self.useable = False + try: + topics_dirs = import_dest_cfg.get_rvalue("topics_dirs") + if topics_dirs[0] and os.path.isdir(topics_dirs[0]): + self.useable = True + self._dest["topics_dirs"] = topics_dirs[0] + except RMTException: + self.useable = False + self._wb = None + tracer.debug("Finished.") + + def run(self): + if self.useable: + filename = self._cfg["import_filename"] + if filename and os.path.isfile(filename): + self.import_file(filename) + + def import_file(self, filename): + self._wb = openpyxl.load_workbook(filename) + headers, self._entries, self._topics = self._extract_dict() + self._entries = self._verify_entries(self._entries) + self._write_to_files(self._entries, self._topics) + self._post_process_file(filename) + + def _extract_dict(self): + headers = None + entries = [] + req_wb = self._wb[self._cfg["requirement_ws"]] + for row in req_wb: + if row[0].value == "ID": + headers = [cell.value for cell in row] + elif headers and row[0].value: + req = OrderedDict([(headers[i], cell.value) + for i, cell in enumerate(row)]) + entries.append(req) + + topics = OrderedDict() + topics_ws = self._wb[self._cfg["topics_sheet"]] + for row in topics_ws.rows: + if row[0].value: + topic_name = row[0].value + topics[topic_name] = [] + if len(row) >= 3 and row[1].value: + topics[topic_name].append((row[1].value, row[2].value)) + + return headers, entries, topics + + @staticmethod + def _verify_entries(entries): + """Verify requirement id are not twice in the list. It could easily a + copy-paste error + + Future inventedOn dates causes errors. + + """ + ids = [] + for entry in entries: + assert entry["ID"] not in ids + ids.append(entry["ID"]) + + for entry in entries: + if (entry.get("Invented on", datetime.datetime.now()) > + datetime.datetime.now()): + raise RMTException(118, "Future inventedOn not accepted") + + # add unsolved entries to first + # Add first to list to avoid self-solving + solved_entries = [entries[0]["ID"]] + for entry in entries: + solved_str = entry.get("Solved by", "") + if solved_str: + solved = solved_str.split(" ") + solved_purged = [x for x in solved if x.strip()] + solved_entries.extend(solved_purged) + + added_entries = [] + for entry in entries: + if entry["ID"] not in solved_entries: + added_entries.append(entry["ID"]) + if added_entries: + if entries[0]["Solved by"]: + entries[0]["Solved by"] = (entries[0]["Solved by"].strip() + + " " + " ".join(added_entries)) + else: + entries[0]["Solved by"] = " ".join(added_entries) + return entries + + def _write_to_files(self, entries, topics): + for entry in entries: + filepath = os.path.join(self._dest["requirements_dirs"], + entry["ID"] + ".req") + with codecs.open(filepath, "w", "utf-8") as fhdl: + for key, value in entry.items(): + content = None + if key == "ID": + pass + elif isinstance(value, datetime.date): + content = str(value.date()) + elif value: + content = "\n ".join(str(value).splitlines()) + if content: + fhdl.write(": ".join([key, content]) + os.linesep) + + for name in topics.keys(): + filepath = os.path.join(self._dest["topics_dirs"], name + ".tic") + with codecs.open(filepath, "w", "utf-8") as fhdl: + for key, value in topics[name]: + fhdl.write(": ".join([key, value]) + os.linesep) + + @staticmethod + def _post_process_file(filename): + old_file = filename.split(".") + app = datetime.datetime.now().isoformat() + dest_file = old_file[:-1] + [app] + old_file[-1:] + dest = ".".join(dest_file) + distutils.file_util.move_file(filename, dest) diff --git a/rmtoo/lib/Import.py b/rmtoo/lib/Import.py new file mode 100644 index 00000000..93562cd9 --- /dev/null +++ b/rmtoo/lib/Import.py @@ -0,0 +1,87 @@ +"""Import data from other sources and write to the file system entries + +(c) 2018, 2020 by Kristoffer +SPDX-License-Identifier: GPL-3.0-or-later +This file is part of rmtoo. +For licensing details see COPYING + +""" +from __future__ import unicode_literals +from stevedore import extension +from six import iteritems + +from rmtoo.lib.logging import tracer +from rmtoo.lib.configuration.Cfg import Cfg +from rmtoo.imports.abcimports import AbcImports + + +class Import: + """Class parsing the configuration and importing from configured + import locations to configured input location. + + The idea is to not deal with, e.g., Excel sheets in this tool but + to enable seamless import and export. + + The default configuration provides no information about the + location of the *ts_topics.sources* location. + + :ivar _input_dir: latest item with requirements_dir and topics_dir + + :Example: an example configuration looks as follows. + "import": { + "xls": { + "import_filename": "import/requirements.xls" + } + } + + """ + + DEFAULT_CONFIG = {"xls": {"import_filename": None}} + + def __init__(self, config): + """Sets up Import for use.""" + tracer.info("called") + self.__plugin_manager = extension.ExtensionManager( + namespace="rmtoo.imports.plugin", invoke_on_load=False) + + assert config # we need a configuration + if "import" in config: + self._config = config["import"] + else: + self._config = self.DEFAULT_CONFIG + self._cfg = Cfg(self._config) + + self._input_dir = {"requirements_dirs": None, "topics_dirs": None} + self._extract_input_dir(config) + + self._import_obj = [] + self._set_run_modules() + tracer.debug("Finished.") + + def _extract_input_dir(self, config): + input_cfg = Cfg(config) + sources_dirs = "topics.ts_common.sources" + for cfg in input_cfg.get_value(sources_dirs): + for req_idx in [ + "requirements_dirs", "topics_dirs", "sources_dirs" + ]: + if req_idx in cfg[1]: + self._input_dir[req_idx] = cfg[1][req_idx] + + def _set_run_modules(self): + for module_name, cfg in iteritems(self._config): + import_obj = self.__plugin_manager[module_name].plugin( + cfg, self._input_dir) + if isinstance(import_obj, AbcImports): + self._import_obj.append(import_obj) + + @staticmethod + def execute(config): + """Process Import with configuration file""" + foreign_import = Import(config) + foreign_import.process_all() + + def process_all(self): + """Process all members""" + for module in self._import_obj: + module.run() diff --git a/rmtoo/lib/RmtooMain.py b/rmtoo/lib/RmtooMain.py index d24fc073..ab8c5422 100644 --- a/rmtoo/lib/RmtooMain.py +++ b/rmtoo/lib/RmtooMain.py @@ -19,11 +19,14 @@ from rmtoo.lib.TopicContinuumSet import TopicContinuumSet from rmtoo.lib.Analytics import Analytics from rmtoo.lib.Output import Output +from rmtoo.lib.Import import Import from rmtoo.lib.main.MainHelper import MainHelper from rmtoo.lib.logging import configure_logging def execute_cmds(config, input_mods, _mstdout, mstderr): + Import.execute(config) # Import foreign data + '''Checks are always done - to be sure that e.g. the dependencies are correct. Please note: there is no 'ONE' latest continuum any more diff --git a/rmtoo/lib/storagebackend/txtfile/TxtRecordEntry.py b/rmtoo/lib/storagebackend/txtfile/TxtRecordEntry.py index 525600d4..fb12b930 100644 --- a/rmtoo/lib/storagebackend/txtfile/TxtRecordEntry.py +++ b/rmtoo/lib/storagebackend/txtfile/TxtRecordEntry.py @@ -94,3 +94,7 @@ def set_comment(self, comment): def get_content_with_nl(self): """Return the raw content""" return self.content_raw + + def get_content_trimmed_with_nl(self): + """Return the raw content""" + return [x.strip() for x in self.content_raw] diff --git a/rmtoo/outputs/xls.py b/rmtoo/outputs/xls.py new file mode 100644 index 00000000..7ee5e55f --- /dev/null +++ b/rmtoo/outputs/xls.py @@ -0,0 +1,263 @@ +"""Output the content to a XLS sheet. Suits love Excel. + +(c) 2018, 2020 by Kristoffer +SPDX-License-Identifier: GPL-3.0-or-later +This file is part of rmtoo. +For licensing details see COPYING + +""" +from __future__ import unicode_literals + +import openpyxl +import datetime +import shutil + +from rmtoo.lib.StdOutputParams import StdOutputParams +from rmtoo.lib.ExecutorTopicContinuum import ExecutorTopicContinuum +from rmtoo.lib.logging import tracer +from rmtoo.lib.CreateMakeDependencies import CreateMakeDependencies + + +class XlsHandler: + """Act as an abstraction layer between rmtoo-objects and OpenPyxl + related things + + The default configuration is intended to sort the important + objects. Users can adapt it to their needs. + + The filename is handled in the Xls' parent class. + + """ + + default_config = { + "output_filename": + "artifacts/requirements.xlsx", + "req_attributes": [ + "ID", + "Name", + "Topic", + "Description", + "Status", + "Owner", + "Invented by", + "Invented on", + ], + "headers": [ + "ID", + "Name", + "Topic", + "Description", + "Status", + "Rationale", + "Owner", + "Effort estimation", + "Invented on", + ], + "req_sheet": + "Requirements", + "topic_sheet": + "Topics", + } + + def __init__(self, filename, config=None): + tracer.info("Creating XLS workbook: " + filename) + self.__filename = filename + self._cfg = self.default_config + for key, value in config.items(): + self._cfg[key] = value + + # We require those headers at least + self._req_headers = self._cfg["req_attributes"] + self._headers = list(self._cfg["headers"]) + self.req_row = 1 + self._reqlist = [] + self._topiclist = [] + + self._prepare_template() + + def write(self): + self._add_header() + self._write_reqs() + self._write_topics() + self._wb.save(filename=self.__filename) + + def _prepare_template(self): + usable = False + if "template_filename" in self._cfg: + try: + usable = self._template_useable() + except AssertionError: + tracer.warning("Using default template") + if usable: + shutil.copyfile(self._cfg["template_filename"], self.__filename) + self._wb = openpyxl.load_workbook(filename=self.__filename) + self._ws_req = self._wb[self._cfg["req_sheet"]] + self._ws_topics = self._wb[self._cfg["topic_sheet"]] + self._ws_cfg = self._wb["Configuration"] + else: + self._new_workbook() + + def _template_useable(self): + filename = self._cfg["template_filename"] + twb = openpyxl.load_workbook(filename=filename, read_only=True) + ws_req = twb[self._cfg["req_sheet"]] + + req_row = None + for j, row in enumerate(ws_req.iter_rows(min_row=1, max_col=1)): + cell = row[0] + if cell.value == "ID": + req_row = j + 1 + break + assert req_row + + header_row = ws_req[req_row] + headers = list() + for cell in header_row: + headers.append(cell.value) + # We want the required headers in our template and just copy this + # list + assert set(self._headers).issubset(headers) + + self.req_row = req_row + self._headers = headers + return True + + def _new_workbook(self): + self._wb = openpyxl.Workbook() + self._ws_req = self._wb.active + self._ws_req.title = self._cfg["req_sheet"] + self._ws_topics = self._wb.create_sheet(self._cfg["topic_sheet"]) + self._ws_cfg = self._wb.create_sheet("Configuration") + + def _add_header(self): + col = 1 + for val in self._headers: + cell = self._ws_req.cell(column=col, row=self.req_row) + if cell.value != val: + cell.value = val + col += 1 + self.req_row += 1 + + def _write_reqs(self): + for req in self._reqlist: + col = 1 + for key in self._headers: + if key in req: + val = req[key] + if isinstance(val, list): + val = str(val) + self._ws_req.cell(column=col, row=self.req_row, value=val) + col += 1 + self.req_row += 1 + self._reqlist = [] + + def _write_topics(self): + row = 1 + for topic in self._topiclist: + self._ws_topics.cell(column=1, row=row, value=topic.name) + for key, content in topic.members.items(): + self._ws_topics.cell(column=2, row=row, value=key) + self._ws_topics.cell(column=3, row=row, value=content) + row += 1 + self._topiclist = [] + + def add_req(self, req): + req_dict = self._req_extract_dict(req) + self._check_required_fields(req_dict) + self._add_new_headers(req_dict) + self._reqlist.append(req_dict) + + @staticmethod + def _req_extract_dict(req): + req_dict = {"ID": req.get_id()} + for rec in req.record: + if rec.get_tag() in req.values and isinstance( + req.values[rec.get_tag()], datetime.date): + req_dict[rec.get_tag()] = req.values[rec.get_tag()] + else: + req_dict[rec.get_tag()] = "\n".join( + rec.get_content_trimmed_with_nl()) + return req_dict + + def _check_required_fields(self, req_dict): + """Will raise KeyError if required fields aren't available""" + for val in self._req_headers: + if val not in req_dict.keys(): + tracer.warning("Key (" + val + ") error in " + req_dict["ID"]) + assert val in req_dict.keys() + + def _add_new_headers(self, req_dict): + for key in list(req_dict.keys()): + if key not in self._headers: + self._headers.append(key) + + def add_topic(self, topic): + class TopicList: + def __init__(self, name): + self.name = name + self.members = {} + + ctop = TopicList(topic.name) + for i in topic.get_tags(): + ctop.members[i.get_tag()] = i.get_content() + self._topiclist.append(ctop) + + +class Xls(StdOutputParams, ExecutorTopicContinuum, CreateMakeDependencies): + def __init__(self, oconfig): + """Create an openpyxl output object.""" + tracer.info("Called.") + StdOutputParams.__init__(self, oconfig) + CreateMakeDependencies.__init__(self) + self.__ce3set = None + self.__fd = None + self._opiface = XlsHandler(self._output_filename, self._config) + + @staticmethod + def strescape(string): + """Escapes a string: hexifies it.""" + result = "" + for fchar in string: + if ord(fchar) >= 32 and ord(fchar) < 127: + result += fchar + else: + result += "%02x" % ord(fchar) + return result + + def topic_set_pre(self, _topics_set): + pass + + def topic_set_post(self, topic_set): + """Clean up file.""" + tracer.debug("Clean up file.") + self._opiface.write() + tracer.debug("Finished.") + + def topic_pre(self, topic): + """Output one topic.""" + self._opiface.add_topic(topic) + + def topic_post(self, _topic): + """Cleanup things for topic.""" + pass + + def topic_name(self, name): + pass + + def topic_text(self, text): + pass + + def requirement_set_pre(self, rset): + """Prepare the requirements set output.""" + self.__ce3set = rset.get_ce3set() + self.__testcases = rset.get_testcases() + + def requirement_set_sort(self, list_to_sort): + """Sort by id.""" + return sorted(list_to_sort, key=lambda r: r.get_id()) + + def requirement(self, req): + self._opiface.add_req(req) + + def cmad_topic_continuum_pre(self, _): + pass diff --git a/rmtoo/tests/RMTTest-Import/RMTTest-Import.py b/rmtoo/tests/RMTTest-Import/RMTTest-Import.py new file mode 100644 index 00000000..b68f06cc --- /dev/null +++ b/rmtoo/tests/RMTTest-Import/RMTTest-Import.py @@ -0,0 +1,97 @@ +# (c) 2018 Kristoffer Nordstroem, see COPYING +import os +import pytest +import distutils.file_util + +from rmtoo.lib.Import import Import +from rmtoo.imports.xls import XlsImport +from rmtoo.tests.lib.Utils import create_tmp_dir, delete_tmp_dir +from rmtoo.lib.Encoding import Encoding + +LDIR = Encoding.to_unicode(os.path.dirname(os.path.abspath(__file__))) + + +@pytest.fixture(scope='function') +def tmpdir(): + tmpdir = create_tmp_dir() + yield tmpdir + delete_tmp_dir(tmpdir) + + +@pytest.fixture +def def_cfg(tmpdir): + def_cfg_imp_dest = {'topics': {'ts_common': { + 'sources': [['dummydriver', { + 'requirements_dirs': [tmpdir], + 'topics_dirs': [tmpdir] + }]]}}, 'tmpdir': tmpdir} + return def_cfg_imp_dest + + +class RMTTestImport: + '''Test-Class to import the modified artifacts again''' + + def rmttest_invalid_config_parser(self): + '''Just figure out where it blows up ''' + with pytest.raises(AssertionError): + Import(None) + + def rmttest_config_parser_wo_cfg(self, def_cfg): + '''Assert the default configuration is loaded, ignore the import + destination directory configuration''' + cfg = {'import': {}, "dummy": "the universe is mind-bogglingly big"} + cfg.update(def_cfg) + importer = Import(cfg) + assert not importer._config + assert not importer._import_obj + + def rmttest_config_parser_input_dir(self, def_cfg): + importer = Import(def_cfg) + assert (importer._input_dir['requirements_dirs'][0] + == def_cfg['tmpdir']) + assert (importer._input_dir['topics_dirs'][0] + == def_cfg['tmpdir']) + + def rmttest_config_parser(self, def_cfg): + '''Assert the default configuration is loaded, ignore the import + destination directory configuration''' + cfg = {"import": {"xls": { + 'import_filename': "asdf.xls"}}} + cfg.update(def_cfg) + importer = Import(cfg) + assert len(importer._import_obj) == 1 + assert isinstance(importer._import_obj[0], XlsImport) + + def rmttest_config_run_with_default_cfg(self, def_cfg): + tmpdir = def_cfg['tmpdir'] + src_fn = os.path.join(LDIR, "test-reqs.xlsx") + tmp_fn = os.path.join(tmpdir, "test-reqs.xlsx") + distutils.file_util.copy_file(src_fn, tmp_fn) + + cfg = {"import": {"xls": { + 'import_filename': tmp_fn}}} + cfg.update(def_cfg) + importer = Import(cfg) + importer.process_all() + + assert os.path.isfile(os.path.join(tmpdir, 'AutomaticGeneration.req')) + assert os.path.isfile(os.path.join(tmpdir, 'Completed.req')) + assert os.path.isfile(os.path.join(tmpdir, 'TestNewlines.req')) + + def rmttest_error_double_ids(self, def_cfg): + tmpdir = def_cfg['tmpdir'] + src_fn = os.path.join(LDIR, "test-reqs-doubleid.xlsx") + tmp_fn = os.path.join(tmpdir, "test-reqs-doubleid.xlsx") + distutils.file_util.copy_file(src_fn, tmp_fn) + + cfg = {"import": {"xls": { + 'import_filename': tmp_fn}}} + cfg.update(def_cfg) + importer = Import(cfg) + with pytest.raises(AssertionError): + importer.process_all() + + assert not os.path.isfile(os.path.join(tmpdir, + 'AutomaticGeneration.req')) + assert not os.path.isfile(os.path.join(tmpdir, 'Completed.req')) + assert not os.path.isfile(os.path.join(tmpdir, 'TestNewlines.req')) diff --git a/rmtoo/tests/RMTTest-Import/RMTTest-XlsImport.py b/rmtoo/tests/RMTTest-Import/RMTTest-XlsImport.py new file mode 100644 index 00000000..565e40d2 --- /dev/null +++ b/rmtoo/tests/RMTTest-Import/RMTTest-XlsImport.py @@ -0,0 +1,185 @@ +# (c) 2018 Kristoffer Nordstroem, see COPYING +from __future__ import unicode_literals +import os +import re +import codecs +import pytest +import distutils.file_util + +from rmtoo.lib.Requirement import Requirement +from rmtoo.imports.xls import XlsImport +from rmtoo.tests.lib.Utils import create_tmp_dir, delete_tmp_dir +from rmtoo.lib.RMTException import RMTException +from rmtoo.lib.Encoding import Encoding + +LDIR = os.path.dirname(os.path.abspath(__file__)) +LIPSUM = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod " + "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim " + "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea " + "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate " + "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint " + "occaecat cupidatat non proident, sunt in culpa qui officia deserunt " + "mollit anim id est laborum.") + + +@pytest.fixture(scope='function') +def tmpdir(): + tmpdir = create_tmp_dir() + yield tmpdir + delete_tmp_dir(tmpdir) + + +@pytest.fixture +def dest_dir(tmpdir): + dest_dirs = {u'requirements_dirs': [Encoding.to_unicode(tmpdir)], + u'topics_dirs': [Encoding.to_unicode(tmpdir)]} + return dest_dirs + + +class RMTTestXlsImport: + '''Test-Class to import the xls artifact''' + + imp_fn = os.path.join(LDIR, 'test-reqs.xlsx') + config = {u'import_filename': imp_fn, + u'requirement_ws': u'Requirements', + u'topics_sheet': u'Topics'} + + def rmttest_invalid_config_parser(self): + '''Just figure out where it blows up''' + dest_dirs = {u'requirements_dirs': [Encoding.to_unicode(tmpdir)], + u'topics_dirs': [Encoding.to_unicode(tmpdir)]} + importer = XlsImport({}, dest_dirs) + assert not importer.useable + + def rmttest_config_run_with_default_cfg(self, dest_dir): + tmpdir = dest_dir['requirements_dirs'][0] + tmp_fn = os.path.join(tmpdir, 'test-reqs.xlsx') + config = dict(self.config) + distutils.file_util.copy_file(config[u'import_filename'], tmp_fn) + config[u'import_filename'] = tmp_fn + + importer = XlsImport(config, dest_dir) + assert importer.useable + importer.run() + + assert os.path.isfile(os.path.join(tmpdir, 'AutomaticGeneration.req')) + completed_filename = os.path.join(tmpdir, 'Completed.req') + assert os.path.isfile(completed_filename) + + # Assert ID is filename and not written to file + id_occ = [re.findall(r'^ID:', line) + for line in open(completed_filename)] + for i in id_occ: + assert not i + + def rmttest_treat_newlines_correctly(self, dest_dir): + tmpdir = dest_dir['requirements_dirs'][0] + + config = dict(self.config) + tmp_fn = os.path.join(tmpdir, 'test-reqs.xlsx') + distutils.file_util.copy_file(config[u'import_filename'], tmp_fn) + config[u'import_filename'] = tmp_fn + + importer = XlsImport(config, dest_dir) + importer.run() + + newlines_filename = os.path.join(tmpdir, 'TestNewlines.req') + assert os.path.isfile(newlines_filename) + with codecs.open(newlines_filename, encoding='utf-8') as nl_fh: + req_content = nl_fh.read() + nl_req = Requirement(req_content, 'TestNewlines.req', + newlines_filename, None, None) + + # Test Description + parsed_desc = "\n".join(nl_req.record[2]. + get_content_trimmed_with_nl()) + assert parsed_desc == LIPSUM + "\n\nASDF" + + parsed_note = "\n".join(nl_req.record[10]. + get_content_trimmed_with_nl()) + assert parsed_note == "Lipsum\n\nHandle it well" + + parsed_invon = "\n".join(nl_req.record[7]. + get_content_trimmed_with_nl()) + assert parsed_invon == "2010-03-06" + + def rmttest_future_invented_on(self, dest_dir, record_property): + '''Ensure future InventedOn are not imported''' + record_property('req', 'Import/XlsFutureInventedOn-deadbeef') + tmpdir = dest_dir['requirements_dirs'][0] + + lcfg = dict(self.config) + imp_fn = os.path.join(LDIR, 'test-reqs-future.xlsx') + tmp_fn = os.path.join(tmpdir, 'test-reqs-future.xlsx') + distutils.file_util.copy_file(imp_fn, tmp_fn) + lcfg[u'import_filename'] = tmp_fn + + importer = XlsImport(lcfg, dest_dir) + assert importer.useable + with pytest.raises(RMTException): + importer.run() + + def rmttest_set_solvedby(self, dest_dir, record_property): + '''Esnure stuff''' + record_property('req', 'Import/XlsDefaultSolvedby-deadbeef') + tmpdir = dest_dir['requirements_dirs'][0] + + lcfg = dict(self.config) + imp_fn = os.path.join(LDIR, 'test-reqs-solvedby.xlsx') + tmp_fn = os.path.join(tmpdir, 'test-reqs-solvedby.xlsx') + distutils.file_util.copy_file(imp_fn, tmp_fn) + lcfg[u'import_filename'] = tmp_fn + + importer = XlsImport(lcfg, dest_dir) + assert importer.useable + importer.run() + + assert importer._entries[0]['ID'] == 'SW-101' + assert importer._entries[0]['Solved by'] == 'SW-102 SW-104 SW-105' + + def rmttest_defcfg_import_topics(self, dest_dir): + tmpdir = dest_dir['requirements_dirs'][0] + tmp_fn = os.path.join(tmpdir, 'test-reqs.xlsx') + config = dict(self.config) + distutils.file_util.copy_file(config[u'import_filename'], tmp_fn) + config[u'import_filename'] = tmp_fn + + importer = XlsImport(config, dest_dir) + assert importer.useable + importer.run() + + assert importer._topics['GUI'][0] == ('Name', 'GUI') + assert os.path.isfile(os.path.join(tmpdir, 'rmtoo.tic')) + completed_filename = os.path.join(tmpdir, 'GUI.tic') + assert os.path.isfile(completed_filename) + + # Assert ID is filename and not written to file + found = False + id_occ = [re.findall(r'^Name:.*', line) + for line in open(completed_filename)] + for i in id_occ: + if i and i[0] == 'Name: GUI': + found = True + assert found + + +class RMTTestXlsImportRegressionTests: + """Regression tests for the xls import functionality""" + imp_fn = os.path.join(LDIR, 'test-reqs.xlsx') + config = {u'import_filename': imp_fn, + u'requirement_ws': u'Specification', + u'topics_sheet': u'Topics'} + + def rmttest_regression_test_import_empty(self, dest_dir): + imp_fn = os.path.join(LDIR, 'regression-import_empty.xlsx') + tmpdir = dest_dir['requirements_dirs'][0] + tmp_fn = os.path.join(tmpdir, 'regression-import_empty.xlsx') + config = dict(self.config) + config[u'import_filename'] = tmp_fn + distutils.file_util.copy_file(imp_fn, tmp_fn) + + importer = XlsImport(config, dest_dir) + assert importer.useable + importer.run() + assert importer._entries[0]['Solved by'] == 'SW-AS-100 SW-AS-101' diff --git a/rmtoo/tests/RMTTest-Import/regression-import_empty.xlsx b/rmtoo/tests/RMTTest-Import/regression-import_empty.xlsx new file mode 100644 index 00000000..4778df72 Binary files /dev/null and b/rmtoo/tests/RMTTest-Import/regression-import_empty.xlsx differ diff --git a/rmtoo/tests/RMTTest-Import/test-reqs-doubleid.xlsx b/rmtoo/tests/RMTTest-Import/test-reqs-doubleid.xlsx new file mode 100644 index 00000000..a6f5a9a4 Binary files /dev/null and b/rmtoo/tests/RMTTest-Import/test-reqs-doubleid.xlsx differ diff --git a/rmtoo/tests/RMTTest-Import/test-reqs-future.xlsx b/rmtoo/tests/RMTTest-Import/test-reqs-future.xlsx new file mode 100644 index 00000000..8e4ff91f Binary files /dev/null and b/rmtoo/tests/RMTTest-Import/test-reqs-future.xlsx differ diff --git a/rmtoo/tests/RMTTest-Import/test-reqs-solvedby.xlsx b/rmtoo/tests/RMTTest-Import/test-reqs-solvedby.xlsx new file mode 100644 index 00000000..c0be66ed Binary files /dev/null and b/rmtoo/tests/RMTTest-Import/test-reqs-solvedby.xlsx differ diff --git a/rmtoo/tests/RMTTest-Import/test-reqs.xlsx b/rmtoo/tests/RMTTest-Import/test-reqs.xlsx new file mode 100644 index 00000000..3d963337 Binary files /dev/null and b/rmtoo/tests/RMTTest-Import/test-reqs.xlsx differ diff --git a/rmtoo/tests/RMTTest-Output/DefaultTemplate.xlsx b/rmtoo/tests/RMTTest-Output/DefaultTemplate.xlsx new file mode 100644 index 00000000..9bb1efbb Binary files /dev/null and b/rmtoo/tests/RMTTest-Output/DefaultTemplate.xlsx differ diff --git a/rmtoo/tests/RMTTest-Output/RMTTest-Xls.py b/rmtoo/tests/RMTTest-Output/RMTTest-Xls.py new file mode 100644 index 00000000..37c6ff59 --- /dev/null +++ b/rmtoo/tests/RMTTest-Output/RMTTest-Xls.py @@ -0,0 +1,154 @@ +# (c) 2018 Kristoffer Nordstroem, see COPYING + +import os +from datetime import datetime +import openpyxl +import pytest +from unittest.mock import Mock + +from rmtoo.outputs.xls import XlsHandler as xh +from rmtoo.lib.Requirement import Requirement +from rmtoo.tests.lib.Utils import create_tmp_dir, delete_tmp_dir +from rmtoo.tests.lib.TestConfig import TestConfig + +LDIR = os.path.dirname(os.path.abspath(__file__)) + + +def create_req(req_id): + req_txt_items = {} + inv_date = datetime.strptime('1970-01-01', "%Y-%m-%d").date() + req_values = { + 'Name': 'Aston Martin DB5', + 'Topic': 'Escape Routes', + 'Description': "Flying out the roof", + 'Priority': 'development', + 'Owner': '007', + 'Invented on': inv_date.isoformat(), + 'Invented by': 'Q', + 'Status': 'not done', + 'Class': 'requirement' + } + req = Requirement(u"", req_id, None, None, TestConfig()) + for key, value in req_values.items(): + req.record.append(MockRecordEntry(key, value)) + req_txt_items[key] = MockRecordEntry(None, value) + req_txt_items['Invented on'] = inv_date + req.values.update(req_txt_items) + return req + + +class MockRecordEntry: + def __init__(self, tag, txt_str): + self.__txt_tag = tag + self.__txt_string = txt_str + + def get_content(self): + return self.__txt_string + + def get_content_trimmed_with_nl(self): + return [x.strip() for x in self.__txt_string.splitlines()] + + def get_output_string(self): + return self.__txt_string + + def get_tag(self): + return self.__txt_tag + + +class RMTTestOutputXls: + "Test-Class for the xlsx output class" + + def teardown_method(self): + if self.__tmpdir: + delete_tmp_dir(self.__tmpdir) + + def setup_method(self): + self.__tmpdir = create_tmp_dir() + self.oconfig = xh.default_config + self._filename = os.path.join(self.__tmpdir, "reqs.xlsx") + self.oconfig["output_filename"] = self._filename + + def rmttest_adding_req_header(self): + xlsh = xh(self._filename, self.oconfig) + xlsh.write() + + twb = openpyxl.load_workbook(filename=self._filename) + rws = twb['Requirements'] + + i = 0 + for tval in xh.default_config['headers']: + i += 1 + assert rws.cell(row=1, column=i).value == tval + assert i == 9 + assert rws['B1'].value == "Name" + + def rmttest_adding_req(self): + xlsh = xh(self._filename, self.oconfig) + xlsh.add_req(create_req(u'SW-101')) + xlsh.add_req(create_req(u'SW-102')) + xlsh.write() + + twb = openpyxl.load_workbook(filename=self._filename) + rws = twb['Requirements'] + assert rws['A2'].value == "SW-101" + assert rws['G2'].value == "007" + assert rws['I2'].value.date().isoformat() == "1970-01-01" + + assert rws['A3'].value == "SW-102" + + def rmttest_adding_req_with_missing_field(self): + xlsh = xh(self._filename, self.oconfig) + req = create_req(u'SW-101') + del req.values['Invented by'] + for i in range(len(req.record)): + if req.record[i].get_content() == 'Q': + req.record.pop(i) + break + with pytest.raises(AssertionError): + xlsh.add_req(req) + + def rmttest_adding_topic(self): + xlsh = xh(self._filename, self.oconfig) + topic_tags = [Mock(**{'get_tag.return_value': "asdf", + 'get_content.return_value': "qwer"})] + topic_cfg = {'get_tags.return_value': topic_tags} + topic = Mock(**topic_cfg) + topic.name = "SuperTopic" + xlsh.add_topic(topic) + xlsh.write() + + twb = openpyxl.load_workbook(filename=self._filename) + rws = twb['Topics'] + assert rws['A1'].value == "SuperTopic" + assert rws['B1'].value == "asdf" + assert rws['C1'].value == "qwer" + + +class RMTTestOutputXlsTemplate: + '''Test-Class for the xlsx output with a template workbook''' + + def teardown_method(self): + if self.__tmpdir: + delete_tmp_dir(self.__tmpdir) + + def setup_method(self): + self.__tmpdir = create_tmp_dir() + self.oconfig = xh.default_config + self._filename = os.path.join(self.__tmpdir, "reqs.xlsx") + self.oconfig["output_filename"] = self._filename + self.oconfig["template_filename"] = os.path.join( + LDIR, 'DefaultTemplate.xlsx') + + def rmttest_adding_req(self): + xlsh = xh(self._filename, self.oconfig) + xlsh.add_req(create_req(u'SW-101')) + xlsh.add_req(create_req(u'SW-102')) + xlsh.write() + + twb = openpyxl.load_workbook(filename=self._filename) + rws = twb['Requirements'] + assert rws['A3'].value == "SW-101" + assert rws['G3'].value == "007" + assert rws['I3'].value.date().isoformat() == "1970-01-01" + + assert rws['A4'].value == "SW-102" diff --git a/setup.py b/setup.py index 71b8f108..82188947 100644 --- a/setup.py +++ b/setup.py @@ -131,7 +131,9 @@ def adjust(input_filename, output): "stevedore>=1.21", "pylint>=1.7.1", "odfpy==1.3.4", - "jinja2>=2.10"], + "jinja2>=2.10", + "six>=1.17", + "openpyxl>=2.5.6"], license="GPL-3.0-or-later", platforms="all", @@ -185,7 +187,11 @@ def adjust(input_filename, output): "tlp1 = rmtoo.outputs.tlp1:Tlp1", "version1 = rmtoo.outputs.version1:version1", "xml1 = rmtoo.outputs.xml1:Xml1", - "xml_ganttproject_2 = rmtoo.outputs.xml_ganttproject_2:xml_ganttproject_2" + "xml_ganttproject_2 = rmtoo.outputs.xml_ganttproject_2:xml_ganttproject_2", + "xls = rmtoo.outputs.xls:Xls" + ], + "rmtoo.imports.plugin" : [ + "xls = rmtoo.imports.xls:XlsImport" ], "rmtoo.output.markup" : [ "latex = rmtoo.lib.Markup:LaTeX",