From 18d44cdca0032c071f3110c2c6e12478969ebaa3 Mon Sep 17 00:00:00 2001 From: solawing <316786359@qq.com> Date: Thu, 4 Aug 2016 22:56:06 +0800 Subject: [PATCH 1/5] sort sections --- xUnique.py | 219 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 141 insertions(+), 78 deletions(-) diff --git a/xUnique.py b/xUnique.py index cfc98f4..4977e0d 100755 --- a/xUnique.py +++ b/xUnique.py @@ -27,7 +27,6 @@ from fileinput import (input as fi_input, close as fi_close) from re import compile as re_compile from sys import (argv as sys_argv, getfilesystemencoding as sys_get_fs_encoding, version_info) -from collections import deque from filecmp import cmp as filecmp_cmp from optparse import OptionParser @@ -240,98 +239,162 @@ def substitute_old_keys(self): def sort_pbxproj(self, sort_pbx_by_file_name=False): self.vprint('sort project.xpbproj file') - lines = [] removed_lines = [] + files_start_ptn = re_compile('^(\s*)files = \(\s*$') files_key_ptn = re_compile('((?<=[A-Z0-9]{24} \/\* )|(?<=[A-F0-9]{32} \/\* )).+?(?= in )') - fc_end_ptn = '\);' - files_flag = False children_start_ptn = re_compile('^(\s*)children = \(\s*$') children_pbx_key_ptn = re_compile('((?<=[A-Z0-9]{24} \/\* )|(?<=[A-F0-9]{32} \/\* )).+?(?= \*\/)') - child_flag = False - pbx_start_ptn = re_compile('^.*Begin (PBXBuildFile|PBXFileReference) section.*$') - pbx_key_ptn = re_compile('^\s+(([A-Z0-9]{24})|([A-F0-9]{32}))(?= \/\*)') - pbx_end_ptn = ('^.*End ', ' section.*$') - pbx_flag = False - last_two = deque([]) + array_end_ptn = '^{space}\);\s*$' + + pbx_section_start_ptn = re_compile('^\s*\/\*\s*Begin (.+) section.*$') + pbx_section_end_ptn = '^\s*\/\*\s*End {name} section.*$' + pbx_section_names = { + 'PBXGroup', + 'PBXFileReference', + 'PBXBuildFile', + 'PBXContainerItemProxy', + 'PBXReferenceProxy', + 'PBXNativeTarget', + 'PBXTargetDependency', + 'PBXSourcesBuildPhase', + 'PBXFrameworksBuildPhase', + 'PBXResourcesBuildPhase', + 'PBXCopyFilesBuildPhase', + 'PBXShellScriptBuildPhase', + 'XCBuildConfiguration', + 'XCConfigurationList', + 'XCVersionGroup', + 'PBXVariantGroup', + 'PBXProject', + } + pbx_section_names_sort_by_name = {'PBXFileReference', 'PBXBuildFile'} if sort_pbx_by_file_name else set() + pbx_section_item_ptn = re_compile(r'^(\s*){hex_group}\s+{name_group}\s*=\s*\{{{oneline_end_group}\s*$'.format( + hex_group = '((?:[A-Z0-9]{24})|(?:[A-F0-9]{32}))', + name_group = r'(?:\/\* (.+?) \*\/)?', + oneline_end_group = '(?:.+(};))?' + )) + pbx_section_item_end_ptn = r"^{space}\}};\s*$" + + empty_line_ptn = re_compile('^\s*$') def file_dir_order(x): x = children_pbx_key_ptn.search(x).group() return '.' in x, x - for line in fi_input(self.xcode_pbxproj_path, backup='.sbak', inplace=1): - # project.pbxproj is an utf-8 encoded file - line = decoded_string(line, 'utf-8') - last_two.append(line) - if len(last_two) > 2: - last_two.popleft() - # files search and sort + output_stack = [output_u8line] + write = lambda *args: output_stack[-1](*args) + + deal_stack = [] + deal = lambda line: deal_stack[-1](line) + + def check_section(line): + section_match = pbx_section_start_ptn.search(line) + if section_match: + write(line) + section_name = section_match.group(1) + if section_name in pbx_section_names: + section_items = [] + end_ptn = re_compile(pbx_section_end_ptn.format(name=section_name)) + def check_end(line): + end_match = bool(end_ptn.search(line)) + if end_match: + if section_items: + section_items.sort(key=lambda item: item[0]) + write(''.join( i[1] for i in section_items )) + write(line) + deal_stack.pop() + return end_match + section_item_key_group = 3 if section_name in pbx_section_names_sort_by_name else 2 + def deal_section_line(line): + if check_end(line): return + section_item_match = pbx_section_item_ptn.search(line) + if section_item_match: + section_item_key = section_item_match.group(section_item_key_group) + if not section_item_key: section_item_key = "" + if section_item_match.group(4): # oneline item + section_items.append((section_item_key, line)) + else: # multiline item + lines = [line] + end_ptn = re_compile(pbx_section_item_end_ptn.format(space=section_item_match.group(1))) + def check_item_end(line): + end_match = bool(end_ptn.search(line)) + if end_match: + write(line) + section_items.append((section_item_key, ''.join(lines))) + output_stack.pop() + deal_stack.pop() + return end_match + def deal_section_item_line(line): + if check_item_end(line) or check_files(line) or check_children(line): + return + write(line) + output_stack.append(lambda line: lines.append(line)) + deal_stack.append(deal_section_item_line) + elif empty_line_ptn.search(line): pass + else: raise XUniqueExit("unexpected line:\n{}".format(line)) + deal_stack.append(deal_section_line) + return True + return False + def check_files(line): files_match = files_start_ptn.search(line) if files_match: - output_u8line(line) - files_flag = True - if isinstance(fc_end_ptn, six.text_type): - fc_end_ptn = re_compile(files_match.group(1) + fc_end_ptn) - if files_flag: - if fc_end_ptn.search(line): - if lines: - lines.sort(key=lambda file_str: files_key_ptn.search(file_str).group()) - output_u8line(''.join(lines)) - lines = [] - files_flag = False - fc_end_ptn = '\);' - elif files_key_ptn.search(line): - if line in lines: - removed_lines.append(line) - else: - lines.append(line) - # children search and sort + write(line) + lines = [] + end_ptn = re_compile(array_end_ptn.format(space=files_match.group(1))) + def deal_files(line): + if end_ptn.search(line): + if lines: + lines.sort(key=lambda file_str: files_key_ptn.search(file_str).group()) + write(''.join(lines)) + write(line) + deal_stack.pop() + elif files_key_ptn.search(line): + if line in lines: removed_lines.append(line) + else: lines.append(line) + elif empty_line_ptn.search(line): pass + else: raise XUniqueExit("unexpected line:\n{}".format(line)) + deal_stack.append(deal_files) + return True + return False + def check_children(line): children_match = children_start_ptn.search(line) if children_match: - output_u8line(line) - child_flag = True - if isinstance(fc_end_ptn, six.text_type): - fc_end_ptn = re_compile(children_match.group(1) + fc_end_ptn) - if child_flag: - if fc_end_ptn.search(line): - if lines: - if self.main_group_hex not in last_two[0]: + write(line) + lines = [] + end_ptn = re_compile(array_end_ptn.format(space=children_match.group(1))) + def deal_children(line): + if end_ptn.search(line): + if lines: lines.sort(key=file_dir_order) - output_u8line(''.join(lines)) - lines = [] - child_flag = False - fc_end_ptn = '\);' - elif children_pbx_key_ptn.search(line): - if line in lines: - removed_lines.append(line) - else: - lines.append(line) - # PBX search and sort - pbx_match = pbx_start_ptn.search(line) - if pbx_match: - output_u8line(line) - pbx_flag = True - if isinstance(pbx_end_ptn, tuple): - pbx_end_ptn = re_compile(pbx_match.group(1).join(pbx_end_ptn)) - if pbx_flag: - if pbx_end_ptn.search(line): - if lines: - if sort_pbx_by_file_name: - lines.sort(key=lambda file_str: children_pbx_key_ptn.search(file_str).group()) - else: - lines.sort(key=lambda file_str: pbx_key_ptn.search(file_str).group(1)) - output_u8line(''.join(lines)) - lines = [] - pbx_flag = False - pbx_end_ptn = ('^.*End ', ' section.*') - elif children_pbx_key_ptn.search(line): - if line in lines: - removed_lines.append(line) - else: - lines.append(line) - # normal output - if not (files_flag or child_flag or pbx_flag): - output_u8line(line) + write(''.join(lines)) + write(line) + deal_stack.pop() + elif children_pbx_key_ptn.search(line): + if line in lines: removed_lines.append(line) + else: lines.append(line) + elif empty_line_ptn.search(line): pass + else: raise XUniqueExit("unexpected line:\n{}".format(line)) + deal_stack.append(deal_children) + return True + return False + def deal_global_line(line): + if check_section(line) or check_files(line) or check_children(line): + return + write(line) + deal_stack.append(deal_global_line) + try: + for line in fi_input(self.xcode_pbxproj_path, backup='.sbak', inplace=1): + # project.pbxproj is an utf-8 encoded file + line = decoded_string(line, 'utf-8') + deal(line) + assert len(deal_stack) == 1 and len(output_stack) == 1 + except Exception as e: + fi_close() + tmp_path = self.xcode_pbxproj_path + '.sbak' + unlink(self.xcode_pbxproj_path) + rename(tmp_path, self.xcode_pbxproj_path) + raise e fi_close() tmp_path = self.xcode_pbxproj_path + '.sbak' if filecmp_cmp(self.xcode_pbxproj_path, tmp_path, shallow=False): From 73c72f85cec4f81aa8bb0cbd9a5b4aa475a31b73 Mon Sep 17 00:00:00 2001 From: solawing <316786359@qq.com> Date: Fri, 5 Aug 2016 14:55:35 +0800 Subject: [PATCH 2/5] FIX: possible conflict with hash --- xUnique.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/xUnique.py b/xUnique.py index 4977e0d..fc23436 100755 --- a/xUnique.py +++ b/xUnique.py @@ -106,11 +106,13 @@ def __init__(self, target_path, verbose=False): self.root_node = self.nodes[self.root_hex] self.main_group_hex = self.root_node['mainGroup'] self.__result = {} + root_new_hex = md5_hex(self.proj_root) + self.__new_key_path_dict = {root_new_hex : self.proj_root} # initialize root content self.__result.update( { self.root_hex: {'path': self.proj_root, - 'new_key': md5_hex(self.proj_root), + 'new_key': root_new_hex, 'type': self.root_node['isa'] } }) @@ -132,6 +134,18 @@ def pbxproj_to_json(self): 2. The project file is not broken, such like merge conflicts, incomplete content due to xUnique failure. """.format( cpe.output)) + def __update_result(self, current_hex, path, new_key, atype): + old = self.__result.get(current_hex) + if old: + self.vprint("override", current_hex) + self.__new_key_path_dict.pop(old['new_key'], None) + while new_key in self.__new_key_path_dict and self.__new_key_path_dict[new_key] != path: + self.vprint("hash conflicts old:{} => new:{}".format(current_hex, new_key)) + new_key = md5_hex(new_key) # rehash to avoid conflicts of different path + self.__new_key_path_dict[new_key] = path + self.__result[current_hex] = {'path': path, 'new_key': new_key, 'type': atype} + return new_key + def __set_to_result(self, parent_hex, current_hex, current_path_key): current_node = self.nodes[current_hex] isa_type = current_node['isa'] @@ -146,13 +160,7 @@ def __set_to_result(self, parent_hex, current_hex, current_path_key): raise KeyError('current_path_key must be list/tuple/string') cur_abs_path = '{}/{}'.format(self.__result[parent_hex]['path'], current_path) new_key = md5_hex(cur_abs_path) - self.__result.update({ - current_hex: {'path': '{}[{}]'.format(isa_type, cur_abs_path), - 'new_key': new_key, - 'type': isa_type - } - }) - return new_key + return self.__update_result(current_hex, '{}[{}]'.format(isa_type, cur_abs_path), new_key, isa_type) def get_proj_root(self): """PBXProject name,the root node""" @@ -501,13 +509,9 @@ def __unique_container_item_proxy(self, parent_hex, container_item_proxy_hex): else: portal_path = portal_result_hex['path'] new_rg_id_path = '{}+{}'.format(cur_path, portal_path) - self.__result.update({ - remote_global_id_hex: {'path': new_rg_id_path, - 'new_key': md5_hex(new_rg_id_path), - 'type': '{}#{}'.format(self.nodes[container_item_proxy_hex]['isa'], - 'remoteGlobalIDString') - } - }) + self.__update_result(remote_global_id_hex, new_rg_id_path, md5_hex(new_rg_id_path), + '{}#{}'.format(self.nodes[container_item_proxy_hex]['isa'], + 'remoteGlobalIDString')) def __unique_build_phase(self, parent_hex, build_phase_hex): """PBXSourcesBuildPhase PBXFrameworksBuildPhase PBXResourcesBuildPhase From 6c3c6cde8285043b12c9ca1de1dde33280766e8f Mon Sep 17 00:00:00 2001 From: solawing <316786359@qq.com> Date: Sat, 6 Aug 2016 21:31:36 +0800 Subject: [PATCH 3/5] FIX: remoteGlobalID use subproject's id --- xUnique.py | 57 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/xUnique.py b/xUnique.py index 4977e0d..417797d 100755 --- a/xUnique.py +++ b/xUnique.py @@ -171,6 +171,15 @@ def get_proj_root(self): else: raise XUniqueExit("File 'project.pbxproj' is broken. Cannot find PBXProject name.") + def subproject(self, abspath): + if not hasattr(self, '_subproject'): + self._subproject = {} + sub_proj = self._subproject.get(abspath) + if sub_proj is None: + sub_proj = XUnique(abspath, self.verbose) + self._subproject[abspath] = sub_proj + return sub_proj + def unique_project(self): """iterate all nodes in pbxproj file: @@ -487,27 +496,39 @@ def __unique_container_item_proxy(self, parent_hex, container_item_proxy_hex): current_node = self.nodes[container_item_proxy_hex] # re-calculate remoteGlobalIDString to a new length 32 MD5 digest remote_global_id_hex = current_node.get('remoteGlobalIDString') - if not remote_global_id_hex: - self.__result.setdefault('uniquify_warning', []).append( - "PBXTargetDependency '{}' and its child PBXContainerItemProxy '{}' are not needed anymore, please remove their sections manually".format( - self.__result[parent_hex]['new_key'], new_container_item_proxy_hex)) - elif remote_global_id_hex not in self.__result.keys(): + append_warning = lambda: self.__result.setdefault('uniquify_warning', []).append( + "PBXTargetDependency '{}' and its child PBXContainerItemProxy '{}' are not needed anymore, please remove their sections manually".format( + self.__result[parent_hex]['new_key'], new_container_item_proxy_hex)) + if not remote_global_id_hex: append_warning() + elif remote_global_id_hex not in self.__result: portal_hex = current_node['containerPortal'] portal_result_hex = self.__result.get(portal_hex) - if not portal_result_hex: - self.__result.setdefault('uniquify_warning', []).append( - "PBXTargetDependency '{}' and its child PBXContainerItemProxy '{}' are not needed anymore, please remove their sections manually".format( - self.__result[parent_hex]['new_key'], new_container_item_proxy_hex)) + if not portal_result_hex: append_warning() else: - portal_path = portal_result_hex['path'] - new_rg_id_path = '{}+{}'.format(cur_path, portal_path) - self.__result.update({ - remote_global_id_hex: {'path': new_rg_id_path, - 'new_key': md5_hex(new_rg_id_path), - 'type': '{}#{}'.format(self.nodes[container_item_proxy_hex]['isa'], - 'remoteGlobalIDString') - } - }) + portal = self.nodes[portal_hex] + if portal.get('path'): + abspath = path.join(self.xcodeproj_path, '..', portal['path']) + if abspath == self.xcodeproj_path: return # current project proxy. ignore it + info = current_node.get('remoteInfo') + if info is None: append_warning(); return + + subproject = self.subproject(abspath) + proxyType = int(current_node.get('proxyType', -1)) + if proxyType == 1: + self.__result[remote_global_id_hex] = { + 'new_key': next((v for v in subproject.root_node['targets'] + if subproject.nodes[v]['name'] == info), + remote_global_id_hex)} + elif proxyType == 2: + self.__result[remote_global_id_hex] = { + 'new_key': next((subproject.nodes[v]['productReference'] for v in subproject.root_node['targets'] + if subproject.nodes[v]['name'] == info), + remote_global_id_hex)} + else: # unknown type, ignore it + self.__result.setdefault('uniquify_warning', []).append( + "PBXContainerItemProxy '{}' has unsupported proxyType. don't unique it".format( + remote_global_id_hex)) + self.__result[remote_global_id_hex] = {'new_key': remote_global_id_hex} def __unique_build_phase(self, parent_hex, build_phase_hex): """PBXSourcesBuildPhase PBXFrameworksBuildPhase PBXResourcesBuildPhase From 9d6b145476512138ae256f20db65fb7a2daec2c3 Mon Sep 17 00:00:00 2001 From: solawing <316786359@qq.com> Date: Sun, 7 Aug 2016 10:31:55 +0800 Subject: [PATCH 4/5] FIX: don't sort children in ref project --- xUnique.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/xUnique.py b/xUnique.py index eb98361..1ee3cc2 100755 --- a/xUnique.py +++ b/xUnique.py @@ -294,6 +294,15 @@ def sort_pbxproj(self, sort_pbx_by_file_name=False): pbx_section_item_end_ptn = r"^{space}\}};\s*$" empty_line_ptn = re_compile('^\s*$') + children_nosort_group = set() + try: + # projectReferences may order by xcode, don't sort it + def get_hex(old_hex): + if old_hex in self.__result: return self.__result[old_hex]['new_key'] + return old_hex + for pr in self.root_node['projectReferences']: + children_nosort_group.add(get_hex(pr['ProductGroup'])) + except KeyError as e: pass def file_dir_order(x): x = children_pbx_key_ptn.search(x).group() @@ -334,6 +343,7 @@ def deal_section_line(line): else: # multiline item lines = [line] end_ptn = re_compile(pbx_section_item_end_ptn.format(space=section_item_match.group(1))) + should_sort_children = section_item_match.group(2) not in children_nosort_group def check_item_end(line): end_match = bool(end_ptn.search(line)) if end_match: @@ -343,7 +353,8 @@ def check_item_end(line): deal_stack.pop() return end_match def deal_section_item_line(line): - if check_item_end(line) or check_files(line) or check_children(line): + if check_item_end(line): return + if should_sort_children and (check_files(line) or check_children(line)): return write(line) output_stack.append(lambda line: lines.append(line)) From 6e791085f54fc40a7768da77fff43377a74fa311 Mon Sep 17 00:00:00 2001 From: solawing <316786359@qq.com> Date: Sun, 7 Aug 2016 11:20:36 +0800 Subject: [PATCH 5/5] raise 100 for -c option and modified. this distinguish from exception --- xUnique.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xUnique.py b/xUnique.py index 1ee3cc2..2ca768f 100755 --- a/xUnique.py +++ b/xUnique.py @@ -643,7 +643,7 @@ def main(): parser.add_option("-s", "--sort", action="store_true", dest="sort_bool", default=False, help="sort the project file. default is False. When neither '-u' nor '-s' option exists, xUnique will invisibly add both '-u' and '-s' in arguments") parser.add_option("-c", "--combine-commit", action="store_true", dest="combine_commit", default=False, - help="When project file was modified, xUnique quit with non-zero status. Without this option, the status code would be zero if so. This option is usually used in Git hook to submit xUnique result combined with your original new commit.") + help="When project file was modified, xUnique quit with 100 status. Without this option, the status code would be zero if so. This option is usually used in Git hook to submit xUnique result combined with your original new commit.") parser.add_option("-p", "--sort-pbx-by-filename", action="store_true", dest="sort_pbx_fn_bool", default=False, help="sort PBXFileReference and PBXBuildFile sections in project file, ordered by file name. Without this option, ordered by MD5 digest, the same as Xcode does.") (options, args) = parser.parse_args(sys_argv[1:]) @@ -667,7 +667,8 @@ def main(): xunique.sort_pbxproj(options.sort_pbx_fn_bool) if options.combine_commit: if xunique.is_modified: - raise XUniqueExit("File 'project.pbxproj' was modified, please add it and then commit.") + warning_print("File 'project.pbxproj' was modified, please add it and then commit.") + raise SystemExit(100) else: if xunique.is_modified: warning_print(