Skip to content

[package-deps-hash] Migrate from git ls-tree to git ls-files #5500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dmichon-msft merged 3 commits into from
Dec 12, 2025
Merged

[package-deps-hash] Migrate from git ls-tree to git ls-files #5500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
609ef01
[package-deps-hash] Migrate to git ls-files
dmichon-msft Dec 11, 2025
cd7cd60
Use constants
dmichon-msft Dec 11, 2025
7931915
constant-ify
dmichon-msft Dec 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions common/changes/@microsoft/rush/git-ls-files_2025-12-11-23-00.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "Log a warning if Git-tracked symbolic links are encountered during repo state analysis.",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
10 changes: 10 additions & 0 deletions common/changes/@rushstack/package-deps-hash/git-ls-files_2025-12-11-23-00.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@rushstack/package-deps-hash",
"comment": "Replace \"git ls-tree\" with \"git ls-files\" to improve performance. Identify symbolic links and return them separately in \"getDetailedRepoStateAsync\". Symbolic links will be omitted from the result returned by \"getRepoStateAsync\", as they are not \"files\".",
"type": "minor"
}
],
"packageName": "@rushstack/package-deps-hash"
}
1 change: 1 addition & 0 deletions common/reviews/api/package-deps-hash.api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export interface IDetailedRepoState {
files: Map<string, string>;
hasSubmodules: boolean;
hasUncommittedChanges: boolean;
symlinks: Map<string, string>;
}

// @beta
Expand Down
55 changes: 39 additions & 16 deletions libraries/package-deps-hash/src/getRepoState.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,28 @@ const STANDARD_GIT_OPTIONS: readonly string[] = [
'maintenance.auto=false'
];

const OBJECTMODE_SUBMODULE: '160000' = '160000';
const OBJECTMODE_SYMLINK: '120000' = '120000';
const OBJECTMODE_FILE_NONEXECUTABLE: '100644' = '100644';
const OBJECTMODE_FILE_EXECUTABLE: '100755' = '100755';

// Note that `type` is a stub that is being ignored by the parser in favor of using `mode` to infer, since `%(objecttype)` requires git 2.51.0+
// e.g. 10644 blob <hash>\t<path>
const GIT_LSTREE_FORMAT: string = '%(objectmode) type %(objectname)%x09%(path)';

interface IGitTreeState {
files: Map<string, string>; // type "blob"
symlinks: Map<string, string>; // type "link"
submodules: Map<string, string>; // type "commit"
}

/**
* Parses the output of the "git ls-tree -r -z" command
* Parses the output of the "git ls-tree -r -z" command or of other commands that have been coerced to match its format.
* @internal
*/
export function parseGitLsTree(output: string): IGitTreeState {
const files: Map<string, string> = new Map();
const symlinks: Map<string, string> = new Map();
const submodules: Map<string, string> = new Map();

// Parse the output
Expand All @@ -57,16 +68,21 @@ export function parseGitLsTree(output: string): IGitTreeState {
// The newHash will be all zeros if the file is deleted, or a hash if it exists
const hash: string = item.slice(tabIndex - 40, tabIndex);

const spaceIndex: number = item.lastIndexOf(' ', tabIndex - 42);

const type: string = item.slice(spaceIndex + 1, tabIndex - 41);
const mode: string = item.slice(0, item.indexOf(' '));

switch (type) {
case 'commit': {
switch (mode) {
case OBJECTMODE_SUBMODULE: {
// This is a submodule
submodules.set(filePath, hash);
break;
}
case 'blob':
case OBJECTMODE_SYMLINK: {
// This is a symbolic link
symlinks.set(filePath, hash);
break;
}
case OBJECTMODE_FILE_NONEXECUTABLE:
case OBJECTMODE_FILE_EXECUTABLE:
default: {
files.set(filePath, hash);
break;
Expand All @@ -79,6 +95,7 @@ export function parseGitLsTree(output: string): IGitTreeState {

return {
files,
symlinks,
submodules
};
}
Expand Down Expand Up @@ -386,6 +403,10 @@ export interface IDetailedRepoState {
* The Git file hashes for all files in the repository, including uncommitted changes.
*/
files: Map<string, string>;
/**
* The Git file hashes for all symbolic links in the repository, including uncommitted changes.
*/
symlinks: Map<string, string>;
/**
* A boolean indicating whether the repository has submodules.
*/
Expand Down Expand Up @@ -413,15 +434,15 @@ export async function getDetailedRepoStateAsync(
const statePromise: Promise<IGitTreeState> = spawnGitAsync(
gitPath,
STANDARD_GIT_OPTIONS.concat([
'ls-tree',
// Recursively expand trees
'-r',
'ls-files',
// Read from the index only
'--cached',
// Use NUL as the separator
'-z',
// Specify the full path to files relative to the root
'--full-name',
// As of last commit
'HEAD',
// Match the format of "git ls-tree". The %(objecttype) placeholder requires git 2.51.0+, so not using yet.
`--format=${GIT_LSTREE_FORMAT}`,
'--',
...(filterPath ?? [])
]),
Expand Down Expand Up @@ -454,13 +475,14 @@ export async function getDetailedRepoStateAsync(
}
}

const [{ files }, locallyModified] = await Promise.all([statePromise, locallyModifiedPromise]);
const [{ files, symlinks }, locallyModified] = await Promise.all([statePromise, locallyModifiedPromise]);

for (const [filePath, exists] of locallyModified) {
if (exists) {
if (exists && !symlinks.has(filePath)) {
yield filePath;
} else {
files.delete(filePath);
symlinks.delete(filePath);
}
}
}
Expand All @@ -471,7 +493,7 @@ export async function getDetailedRepoStateAsync(
gitPath
);

const [{ files, submodules }, locallyModifiedFiles] = await Promise.all([
const [{ files, symlinks, submodules }, locallyModifiedFiles] = await Promise.all([
statePromise,
locallyModifiedPromise
]);
Expand Down Expand Up @@ -502,7 +524,8 @@ export async function getDetailedRepoStateAsync(
return {
hasSubmodules,
hasUncommittedChanges: locallyModifiedFiles.size > 0,
files
files,
symlinks
};
}

Expand Down
29 changes: 24 additions & 5 deletions libraries/package-deps-hash/src/test/getRepoDeps.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,29 @@ describe(parseGitLsTree.name, () => {
const hash: string = '3451bccdc831cb43d7a70ed8e628dcf9c7f888c8';

const output: string = `100644 blob ${hash}\t${filename}\x00`;
const { files } = parseGitLsTree(output);
const { files, symlinks, submodules } = parseGitLsTree(output);

expect(symlinks.size).toEqual(0); // Expect there to be exactly 0 symlinks
expect(submodules.size).toEqual(0); // Expect there to be exactly 0 submodules

expect(files.size).toEqual(1); // Expect there to be exactly 1 change
expect(files.get(filename)).toEqual(hash); // Expect the hash to be ${hash}
});

it('can handle a symlink', () => {
const filename: string = 'src/symlink';
const hash: string = '3451bccdc831cb43d7a70ed8e628dcf9c7f888c8';

const output: string = `120000 link ${hash}\t${filename}\x00`;
const { files, symlinks, submodules } = parseGitLsTree(output);

expect(files.size).toEqual(0); // Expect there to be exactly 0 files
expect(submodules.size).toEqual(0); // Expect there to be exactly 0 submodules

expect(symlinks.size).toEqual(1); // Expect there to be exactly 1 symlink
expect(symlinks.get(filename)).toEqual(hash); // Expect the hash to be ${hash}
});

it('can handle a submodule', () => {
const filename: string = 'rushstack';
const hash: string = 'c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac';
Expand All @@ -78,14 +95,16 @@ describe(parseGitLsTree.name, () => {
const filename3: string = 'submodule/src/index.ts';
const hash3: string = 'fedcba9876543210fedcba9876543210fedcba98';

const output: string = `100644 blob ${hash1}\t${filename1}\x00100666 blob ${hash2}\t${filename2}\x00106666 commit ${hash3}\t${filename3}\0`;
const { files, submodules } = parseGitLsTree(output);
const output: string = `100644 blob ${hash1}\t${filename1}\x00100666 blob ${hash2}\t${filename2}\x00160000 commit ${hash3}\t${filename3}\0`;
const { files, symlinks, submodules } = parseGitLsTree(output);

expect(files.size).toEqual(2); // Expect there to be exactly 2 changes
expect(files.size).toEqual(2); // Expect there to be exactly 2 files
expect(files.get(filename1)).toEqual(hash1); // Expect the hash to be ${hash1}
expect(files.get(filename2)).toEqual(hash2); // Expect the hash to be ${hash2}

expect(submodules.size).toEqual(1); // Expect there to be exactly 1 submodule changes
expect(symlinks.size).toEqual(0); // Expect there to be exactly 0 symlink changes

expect(submodules.size).toEqual(1); // Expect there to be exactly 1 submodule
expect(submodules.get(filename3)).toEqual(hash3); // Expect the hash to be ${hash3}
});
});
Expand Down
3 changes: 2 additions & 1 deletion libraries/rush-lib/src/cli/test/RushCommandLineParser.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ jest.mock(`@rushstack/package-deps-hash`, () => {
return {
hasSubmodules: false,
hasUncommittedChanges: false,
files: new Map([['common/config/rush/npm-shrinkwrap.json', 'hash']])
files: new Map([['common/config/rush/npm-shrinkwrap.json', 'hash']]),
symlinks: new Map()
};
},
getRepoChangesAsync(): ReadonlyMap<string, string> {
Expand Down
3 changes: 2 additions & 1 deletion libraries/rush-lib/src/cli/test/RushCommandLineParserFailureCases.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ jest.mock(`@rushstack/package-deps-hash`, () => {
return {
hasSubmodules: false,
hasUncommittedChanges: false,
files: new Map()
files: new Map(),
symlinks: new Map()
};
},
getRepoChangesAsync(): ReadonlyMap<string, string> {
Expand Down
11 changes: 9 additions & 2 deletions libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ export class ProjectChangeAnalyzer {

return async function tryGetSnapshotAsync(): Promise<IInputsSnapshot | undefined> {
try {
const [{ files: hashes, hasUncommittedChanges }, additionalFiles] = await Promise.all([
const [{ files: hashes, symlinks, hasUncommittedChanges }, additionalFiles] = await Promise.all([
getDetailedRepoStateAsync(rootDirectory, additionalRelativePathsToHash, gitPath, filterPath),
getAdditionalFilesFromRushProjectConfigurationAsync(
additionalGlobs,
Expand All @@ -315,8 +315,15 @@ export class ProjectChangeAnalyzer {
)
]);

if (symlinks.size > 0) {
terminal.writeWarningLine(
`Warning: Detected ${symlinks.size} Git-tracked symlinks in the repository. ` +
`These will be ignored by the change detection engine.`
);
}

for (const file of additionalFiles) {
if (hashes.has(file)) {
if (hashes.has(file) || symlinks.has(file)) {
additionalFiles.delete(file);
}
}
Expand Down
3 changes: 2 additions & 1 deletion libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ jest.mock(`@rushstack/package-deps-hash`, () => {
return {
hasSubmodules: false,
hasUncommittedChanges: false,
files: mockHashes
files: mockHashes,
symlinks: new Map()
};
},
getRepoChangesAsync(): ReadonlyMap<string, string> {
Expand Down
Loading