Implement namespaceOverridePaths for Fragmented Package Support

.prettierrc

@@ -4,6 +4,7 @@
     "useTabs": false,
     "printWidth": 120,
     "endOfLine": "auto",
+    "trailingComma": "es5",
     "overrides": [
         {
             "files": ["*.yml", "*.yaml"],

docs/configuration.md

@@ -32,6 +32,8 @@ The following settings control the *environment* in which Pyright will check for
 
 - **extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files.
 
+- **namespaceOverridePaths** [array of strings, optional]: Paths that should be treated as "transparent" namespace packages. If an `__init__.py` file is found within one of these paths, it is ignored, and the import resolver continues to search other import roots for additional fragments of the package. This is useful in environments like Bazel where a single logical package may be split across multiple physical directories, some of which contain an `__init__.py` file. It can also be used for legacy `pkgutil` or `pkg_resources` namespaces where an `__init__.py` file is present but intended to allow merging with other directories.
+
 - **pythonVersion** [string, optional]: Specifies the version of Python that will be used to execute the source code. The version should be specified as a string in the format "M.m" where M is the major version and m is the minor (e.g. `"3.0"` or `"3.6"`). If a version is provided, pyright will generate errors if the source code makes use of language features that are not supported in that version. It will also tailor its use of type stub files, which conditionalizes type definitions based on the version. If no version is specified, pyright will use the version of the current python interpreter, if one is present.
 
 - **pythonPlatform** [string, optional]: Specifies the target platform that will be used to execute the source code. Should be one of `"Windows"`, `"Darwin"`, `"Linux"`, or `"All"`. If specified, pyright will tailor its use of type stub files, which conditionalize type definitions based on the platform. If no platform is specified, pyright will use the current platform.
@@ -246,6 +248,8 @@ The following settings can be specified for each execution environment. Each sou
 
 - **extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each file’s execution environment mapping is independent, so if file A is in one execution environment and imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment.
 
+- **namespaceOverridePaths** [array of strings, optional]: Paths that should be treated as "transparent" namespace packages. If specified, this overrides the default namespaceOverridePaths setting for files within this execution environment. This is useful in environments like Bazel or for legacy `pkgutil` or `pkg_resources` namespaces where an `__init__.py` file is present but intended to allow merging with other directories.
+
 - **pythonVersion** [string, optional]: The version of Python used for this execution environment. If not specified, the global `pythonVersion` setting is used instead.
 
 - **pythonPlatform** [string, optional]: Specifies the target platform that will be used for this execution environment. If not specified, the global `pythonPlatform` setting is used instead.

docs/settings.md

@@ -24,6 +24,8 @@ The Pyright language server honors the following settings.
 
 **python.analysis.extraPaths** [array of paths]: Paths to add to the default execution environment extra paths if there are no execution environments defined in the config file.
 
+**python.analysis.namespaceOverridePaths** [array of paths]: Paths to add to the default execution environment namespace override paths if there are no execution environments defined in the config file.
+
 **python.analysis.ignore** [array of paths]: Paths of directories or files whose diagnostic output (errors and warnings) should be suppressed. This can be overridden in the configuration file.
 
 **python.analysis.include** [array of paths]: Paths of directories or files that should be included. This can be overridden in the configuration file.

packages/pyright-internal/src/analyzer/importResolver.ts

@@ -572,6 +572,12 @@ export class ImportResolver {
             return importResult;
         }
 
+        // If it was shadowed by a regular package, we should not proceed to local imports
+        // (parent directory resolution). This maintains standard shadowing behavior.
+        if (importResult.isShadowed) {
+            return importResult;
+        }
+
         // If the import is absolute and no other method works, try resolving the
         // absolute in the importing file's directory, then the parent directory,
         // and so on, until the import root is reached.
@@ -975,6 +981,7 @@ export class ImportResolver {
             isNamespacePackage: false,
             isInitFilePresent: false,
             isStubPackage: false,
+            isShadowed: false,
             importFailureInfo: importLogger?.getLogs(),
             resolvedUris: [],
             importType: ImportType.Local,
@@ -1356,6 +1363,14 @@ export class ImportResolver {
         };
     }
 
+    private _isNamespaceOverridePath(uri: Uri, execEnv: ExecutionEnvironment): boolean {
+        if (!execEnv.namespaceOverridePaths || execEnv.namespaceOverridePaths.length === 0) {
+            return false;
+        }
+
+        return execEnv.namespaceOverridePaths.some((path) => uri.equals(path));
+    }
+
     private _invalidateFileSystemCache() {
         this._cachedEntriesForPath.clear();
         this._cachedFilesForPath.clear();
@@ -1400,11 +1415,23 @@ export class ImportResolver {
 
             if (allowPyi && this.fileExistsCached(pyiFilePath)) {
                 importLogger?.log(`Resolved import with file '${pyiFilePath}'`);
-                resolvedPaths.push(pyiFilePath);
-                isStubFile = true;
+                if (this._isNamespaceOverridePath(dirPath, execEnv)) {
+                    importLogger?.log(`Treating as namespace package due to namespaceOverridePaths`);
+                    resolvedPaths.push(Uri.empty());
+                    isNamespacePackage = true;
+                } else {
+                    resolvedPaths.push(pyiFilePath);
+                    isStubFile = true;
+                }
             } else if (this.fileExistsCached(pyFilePath)) {
                 importLogger?.log(`Resolved import with file '${pyFilePath}'`);
-                resolvedPaths.push(pyFilePath);
+                if (this._isNamespaceOverridePath(dirPath, execEnv)) {
+                    importLogger?.log(`Treating as namespace package due to namespaceOverridePaths`);
+                    resolvedPaths.push(Uri.empty());
+                    isNamespacePackage = true;
+                } else {
+                    resolvedPaths.push(pyFilePath);
+                }
             } else {
                 importLogger?.log(`Partially resolved import with directory '${dirPath}'`);
                 resolvedPaths.push(Uri.empty());
@@ -1413,6 +1440,8 @@ export class ImportResolver {
 
             implicitImports = this.findImplicitImports(importName, dirPath, [pyFilePath, pyiFilePath]);
         } else {
+            let lastFoundDirectory: Uri | undefined;
+
             for (let i = 0; i < moduleDescriptor.nameParts.length; i++) {
                 const isFirstPart = i === 0;
                 const isLastPart = i === moduleDescriptor.nameParts.length - 1;
@@ -1429,6 +1458,7 @@ export class ImportResolver {
                     if (isFirstPart) {
                         packageDirectory = dirPath;
                     }
+                    lastFoundDirectory = dirPath;
 
                     // See if we can find an __init__.py[i] in this directory.
                     const pyFilePath = dirPath.initPyUri;
@@ -1437,15 +1467,23 @@ export class ImportResolver {
 
                     if (allowPyi && this.fileExistsCached(pyiFilePath)) {
                         importLogger?.log(`Resolved import with file '${pyiFilePath}'`);
-                        resolvedPaths.push(pyiFilePath);
-                        if (isLastPart) {
-                            isStubFile = true;
+                        if (this._isNamespaceOverridePath(dirPath, execEnv)) {
+                            importLogger?.log(`Treating as namespace package due to namespaceOverridePaths`);
+                        } else {
+                            resolvedPaths.push(pyiFilePath);
+                            if (isLastPart) {
+                                isStubFile = true;
+                            }
+                            isInitFilePresent = true;
                         }
-                        isInitFilePresent = true;
                     } else if (this.fileExistsCached(pyFilePath)) {
                         importLogger?.log(`Resolved import with file '${pyFilePath}'`);
-                        resolvedPaths.push(pyFilePath);
-                        isInitFilePresent = true;
+                        if (this._isNamespaceOverridePath(dirPath, execEnv)) {
+                            importLogger?.log(`Treating as namespace package due to namespaceOverridePaths`);
+                        } else {
+                            resolvedPaths.push(pyFilePath);
+                            isInitFilePresent = true;
+                        }
                     }
 
                     if (!pyTypedInfo && lookForPyTyped) {
@@ -1458,11 +1496,6 @@ export class ImportResolver {
                             // so continue to look for the next part.
                             continue;
                         }
-
-                        implicitImports = this.findImplicitImports(moduleDescriptor.nameParts.join('.'), dirPath, [
-                            pyFilePath,
-                            pyiFilePath,
-                        ]);
                         break;
                     }
                 }
@@ -1512,7 +1545,6 @@ export class ImportResolver {
                     resolvedPaths.push(Uri.empty());
 
                     if (isLastPart) {
-                        implicitImports = this.findImplicitImports(importName, dirPath, [pyFilePath, pyiFilePath]);
                         isNamespacePackage = true;
                     }
                 }
@@ -1522,6 +1554,17 @@ export class ImportResolver {
                 }
                 break;
             }
+
+            if (lastFoundDirectory && resolvedPaths.length > 0) {
+                const lastResolvedPath = resolvedPaths[resolvedPaths.length - 1];
+                if (lastResolvedPath.isEmpty() || lastResolvedPath.fileName.startsWith('__init__')) {
+                    const resolvedName = moduleDescriptor.nameParts.slice(0, resolvedPaths.length).join('.');
+                    implicitImports = this.findImplicitImports(resolvedName, lastFoundDirectory, [
+                        lastFoundDirectory.initPyUri,
+                        lastFoundDirectory.initPyiUri,
+                    ]);
+                }
+            }
         }
 
         let importFound: boolean;
@@ -1538,6 +1581,7 @@ export class ImportResolver {
             isNamespacePackage,
             isInitFilePresent,
             isStubPackage,
+            isShadowed: false,
             isImportFound: importFound,
             isPartlyResolved,
             importFailureInfo: importLogger?.getLogs(),
@@ -1669,7 +1713,13 @@ export class ImportResolver {
                 allowPyi,
                 /* lookForPyTyped */ false
             );
-            bestResultSoFar = localImport;
+            if (localImport) {
+                bestResultSoFar = localImport;
+                if ((localImport.isImportFound || localImport.isPartlyResolved) && !localImport.isNamespacePackage) {
+                    bestResultSoFar.isShadowed = true;
+                    return bestResultSoFar;
+                }
+            }
         }
 
         for (const extraPath of execEnv.extraPaths) {
@@ -1687,7 +1737,18 @@ export class ImportResolver {
                 allowPyi,
                 /* lookForPyTyped */ false
             );
-            bestResultSoFar = this._pickBestImport(bestResultSoFar, localImport, moduleDescriptor);
+            if (localImport) {
+                const wasShadowed = bestResultSoFar?.isShadowed;
+                bestResultSoFar = this._pickBestImport(bestResultSoFar, localImport, moduleDescriptor);
+                if (wasShadowed) {
+                    bestResultSoFar.isShadowed = true;
+                }
+
+                if ((localImport.isImportFound || localImport.isPartlyResolved) && !localImport.isNamespacePackage) {
+                    bestResultSoFar.isShadowed = true;
+                    break;
+                }
+            }
         }
 
         // Check for a stdlib typeshed file.
@@ -1730,7 +1791,19 @@ export class ImportResolver {
                 if (thirdPartyImport) {
                     thirdPartyImport.importType = ImportType.ThirdParty;
 
+                    const wasShadowed = bestResultSoFar?.isShadowed;
                     bestResultSoFar = this._pickBestImport(bestResultSoFar, thirdPartyImport, moduleDescriptor);
+                    if (wasShadowed) {
+                        bestResultSoFar.isShadowed = true;
+                    }
+
+                    if (
+                        (thirdPartyImport.isImportFound || thirdPartyImport.isPartlyResolved) &&
+                        !thirdPartyImport.isNamespacePackage
+                    ) {
+                        bestResultSoFar.isShadowed = true;
+                        break;
+                    }
                 }
             }
         } else {
@@ -1786,6 +1859,62 @@ export class ImportResolver {
         bestImportSoFar: ImportResult | undefined,
         newImport: ImportResult | undefined,
         moduleDescriptor: ImportedModuleDescriptor
+    ): ImportResult {
+        if (!bestImportSoFar) {
+            return newImport!;
+        }
+
+        if (!newImport) {
+            return bestImportSoFar;
+        }
+
+        const best = this._pickBestImportInternal(bestImportSoFar, newImport, moduleDescriptor);
+        const other = best === bestImportSoFar ? newImport : bestImportSoFar;
+
+        if (
+            best &&
+            other &&
+            (best.isImportFound || best.isPartlyResolved) &&
+            (other.isImportFound || other.isPartlyResolved)
+        ) {
+            if (best.isNamespacePackage || other.isNamespacePackage) {
+                this._mergeImplicitImports(best, other);
+                if (other.isNamespacePackage) {
+                    best.isNamespacePackage = true;
+                }
+            }
+        }
+
+        return best!;
+    }
+
+    private _mergeImplicitImports(dest: ImportResult, src: ImportResult) {
+        if (!src.implicitImports || src.implicitImports.size === 0) {
+            return;
+        }
+
+        if (!dest.implicitImports) {
+            dest.implicitImports = new Map<string, ImplicitImport>();
+        } else if (dest.implicitImports === src.implicitImports) {
+            return;
+        }
+
+        src.implicitImports.forEach((implImport, name) => {
+            if (!dest.implicitImports!.has(name)) {
+                dest.implicitImports!.set(name, implImport);
+            } else {
+                const existing = dest.implicitImports!.get(name)!;
+                if (!existing.isStubFile && implImport.isStubFile) {
+                    dest.implicitImports!.set(name, implImport);
+                }
+            }
+        });
+    }
+
+    private _pickBestImportInternal(
+        bestImportSoFar: ImportResult | undefined,
+        newImport: ImportResult | undefined,
+        moduleDescriptor: ImportedModuleDescriptor
     ) {
         if (!bestImportSoFar) {
             return newImport;

packages/pyright-internal/src/analyzer/importResult.ts

@@ -50,6 +50,9 @@ export interface ImportResult {
     // directory resolved.
     isInitFilePresent: boolean;
 
+    // True if the import was shadowed by a regular package.
+    isShadowed?: boolean;
+
     // Did it resolve to a stub within a stub package?
     isStubPackage: boolean;
 

packages/pyright-internal/src/analyzer/service.ts

@@ -941,6 +941,14 @@ export class AnalyzerService {
             );
         }
 
+        if (!configOptions.defaultNamespaceOverridePaths && commandLineOptions.configSettings.namespaceOverridePaths) {
+            // Convert user-provided namespace override path strings into absolute Uris 
+            // relative to the execution root.
+            configOptions.defaultNamespaceOverridePaths = commandLineOptions.configSettings.namespaceOverridePaths.map(
+                (p) => executionRoot.combinePaths(p)
+            );
+        }
+
         if (configOptions.defaultPythonPlatform === undefined) {
             configOptions.defaultPythonPlatform = commandLineOptions.configSettings.pythonPlatform;
         }

packages/pyright-internal/src/common/commandLineOptions.ts

@@ -88,6 +88,10 @@ export class CommandLineConfigOptions {
     // when user has not explicitly defined execution environments.
     extraPaths?: string[] | undefined;
 
+    // Namespace override paths to add to the default execution environment
+    // when user has not explicitly defined execution environments.
+    namespaceOverridePaths?: string[] | undefined;
+
     // Default type-checking rule set. Should be one of 'off',
     // 'basic', 'standard', or 'strict'.
     typeCheckingMode?: string | undefined;