From 37e85911f2f8cbe7ab01f6fc1c14696783738a2f Mon Sep 17 00:00:00 2001 From: Guy Smoilovsky <611655+guysmoilov@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:18:27 +0200 Subject: [PATCH 1/7] Create gen-pydoc.py --- gen-pydoc.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 gen-pydoc.py diff --git a/gen-pydoc.py b/gen-pydoc.py new file mode 100644 index 00000000..9e4e1ddf --- /dev/null +++ b/gen-pydoc.py @@ -0,0 +1,30 @@ +import pydoc +import pkgutil +import importlib +import sys + +def generate_docs(package_name): + package = importlib.import_module(package_name) + print(f"\n# pydoc documentation for package: {package_name}\n") + print("\n---\n") + + for _, module_name, is_pkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."): + print(f"\n## pydoc of module: {module_name}\n") + print("\n---\n") + + try: + docstring = pydoc.plain(pydoc.render_doc(module_name)) + print(docstring) + except Exception as e: + print(f"Error documenting {module_name}: {e}\n") + + print("\n---\n") + print(f"\nEnd of documentation for module: {module_name}\n") + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python gen-pydoc.py ") + sys.exit(1) + + package_name = sys.argv[1] + generate_docs(package_name) From 97b0c2d9c32b8b2fee4274e7d8091fa4da1f9611 Mon Sep 17 00:00:00 2001 From: Guy Smoilovsky <611655+guysmoilov@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:22:05 +0200 Subject: [PATCH 2/7] Add plaintext docs to build --- .github/workflows/docs-publish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docs-publish.yml b/.github/workflows/docs-publish.yml index 663916ae..6eec1783 100644 --- a/.github/workflows/docs-publish.yml +++ b/.github/workflows/docs-publish.yml @@ -38,6 +38,10 @@ jobs: # name: docs # path: docs/build + # For LLM consumption + - name: Pydoc plaintext + run: python gen-pydoc.py > build/html/plaintext-pydoc.txt + - name: Authenticate with Google uses: "google-github-actions/auth@v2" with: From 2a2e5606c0dffa70fc44916b54fd69d6bf455892 Mon Sep 17 00:00:00 2001 From: Guy Smoilovsky <611655+guysmoilov@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:31:48 +0200 Subject: [PATCH 3/7] Update gen-pydoc.py Support per-module output --- gen-pydoc.py | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/gen-pydoc.py b/gen-pydoc.py index 9e4e1ddf..60f5c8c9 100644 --- a/gen-pydoc.py +++ b/gen-pydoc.py @@ -1,30 +1,45 @@ import pydoc import pkgutil import importlib +import os import sys +import click -def generate_docs(package_name): +def generate_docs(package_name, output_file=None, file_prefix=None): package = importlib.import_module(package_name) - print(f"\n# pydoc documentation for package: {package_name}\n") - print("\n---\n") + + out_f = open(output_file, 'w') if output_file else sys.stdout + + out_f.write(f"\n# pydoc documentation for package: {package_name}\n") + out_f.write("\n---\n") for _, module_name, is_pkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."): - print(f"\n## pydoc of module: {module_name}\n") - print("\n---\n") + out_f.write(f"\n## pydoc of module: {module_name}\n") + out_f.write("\n---\n") try: docstring = pydoc.plain(pydoc.render_doc(module_name)) - print(docstring) + out_f.write(docstring) except Exception as e: - print(f"Error documenting {module_name}: {e}\n") + out_f.write(f"Error documenting {module_name}: {e}\n") + + out_f.write("\n---\n") + out_f.write(f"\nEnd of documentation for module: {module_name}\n") - print("\n---\n") - print(f"\nEnd of documentation for module: {module_name}\n") + if file_prefix: + module_filename = f"{file_prefix}_{module_name.replace('.', '_')}.txt" + with open(module_filename, 'w') as module_file: + module_file.write(docstring) + + if output_file: + out_f.close() + +@click.command() +@click.argument("package_name") +@click.option("--output-file", "-o", default=None, help="File to write full documentation output.") +@click.option("--file-prefix", "-p", default=None, help="Prefix for separate module documentation files.") +def main(package_name, output_file, file_prefix): + generate_docs(package_name, output_file, file_prefix) if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python gen-pydoc.py ") - sys.exit(1) - - package_name = sys.argv[1] - generate_docs(package_name) + main() From 21c4694f79178931b7e4af3a350da5c5f3d13fc5 Mon Sep 17 00:00:00 2001 From: Guy Smoilovsky <611655+guysmoilov@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:56:30 +0200 Subject: [PATCH 4/7] Update docs-publish.yml pydoc generation per module --- .github/workflows/docs-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-publish.yml b/.github/workflows/docs-publish.yml index 6eec1783..2065df84 100644 --- a/.github/workflows/docs-publish.yml +++ b/.github/workflows/docs-publish.yml @@ -40,7 +40,10 @@ jobs: # For LLM consumption - name: Pydoc plaintext - run: python gen-pydoc.py > build/html/plaintext-pydoc.txt + run: | + python gen-pydoc.py -o build/html/dagshub-pydoc.txt -p build/html/dagshub-pydoc dagshub + python gen-pydoc.py -o build/html/dagshub-annotation-converter-pydoc.txt -p build/html/dagshub-annotation-converter-pydoc dagshub-annotation-converter + - name: Authenticate with Google uses: "google-github-actions/auth@v2" From 9a079759836cb53f7856e16c4176339e0296b3a2 Mon Sep 17 00:00:00 2001 From: Guy Smoilovsky <611655+guysmoilov@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:07:49 +0200 Subject: [PATCH 5/7] Rename gen-pydoc.py to docs/gen-pydoc.py --- gen-pydoc.py => docs/gen-pydoc.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gen-pydoc.py => docs/gen-pydoc.py (100%) diff --git a/gen-pydoc.py b/docs/gen-pydoc.py similarity index 100% rename from gen-pydoc.py rename to docs/gen-pydoc.py From 570bbcc964583dc5aa9d0128f03bc95e968a1e03 Mon Sep 17 00:00:00 2001 From: Guy Smoilovsky <611655+guysmoilov@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:08:33 +0200 Subject: [PATCH 6/7] Update docs-publish.yml --- .github/workflows/docs-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-publish.yml b/.github/workflows/docs-publish.yml index 2065df84..8bd3082d 100644 --- a/.github/workflows/docs-publish.yml +++ b/.github/workflows/docs-publish.yml @@ -41,8 +41,8 @@ jobs: # For LLM consumption - name: Pydoc plaintext run: | - python gen-pydoc.py -o build/html/dagshub-pydoc.txt -p build/html/dagshub-pydoc dagshub - python gen-pydoc.py -o build/html/dagshub-annotation-converter-pydoc.txt -p build/html/dagshub-annotation-converter-pydoc dagshub-annotation-converter + python docs/gen-pydoc.py -o build/html/dagshub-pydoc.txt -p build/html/dagshub-pydoc dagshub + python docs/gen-pydoc.py -o build/html/dagshub-annotation-converter-pydoc.txt -p build/html/dagshub-annotation-converter-pydoc dagshub-annotation-converter - name: Authenticate with Google From c82c96e521b48d6814865c28813ad6aa76744c68 Mon Sep 17 00:00:00 2001 From: Guy Smoilovsky <611655+guysmoilov@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:11:05 +0200 Subject: [PATCH 7/7] Lint gen-pydoc.py --- docs/gen-pydoc.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/gen-pydoc.py b/docs/gen-pydoc.py index 60f5c8c9..563b3245 100644 --- a/docs/gen-pydoc.py +++ b/docs/gen-pydoc.py @@ -1,39 +1,40 @@ import pydoc import pkgutil import importlib -import os import sys import click + def generate_docs(package_name, output_file=None, file_prefix=None): package = importlib.import_module(package_name) - + out_f = open(output_file, 'w') if output_file else sys.stdout - + out_f.write(f"\n# pydoc documentation for package: {package_name}\n") out_f.write("\n---\n") - + for _, module_name, is_pkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."): out_f.write(f"\n## pydoc of module: {module_name}\n") out_f.write("\n---\n") - + try: docstring = pydoc.plain(pydoc.render_doc(module_name)) out_f.write(docstring) except Exception as e: out_f.write(f"Error documenting {module_name}: {e}\n") - + out_f.write("\n---\n") out_f.write(f"\nEnd of documentation for module: {module_name}\n") - + if file_prefix: module_filename = f"{file_prefix}_{module_name.replace('.', '_')}.txt" with open(module_filename, 'w') as module_file: module_file.write(docstring) - + if output_file: out_f.close() + @click.command() @click.argument("package_name") @click.option("--output-file", "-o", default=None, help="File to write full documentation output.") @@ -41,5 +42,6 @@ def generate_docs(package_name, output_file=None, file_prefix=None): def main(package_name, output_file, file_prefix): generate_docs(package_name, output_file, file_prefix) + if __name__ == "__main__": main()