Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions Doc/library/argparse.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1322,20 +1322,25 @@ attribute is determined by the ``dest`` keyword argument of

For optional argument actions, the value of ``dest`` is normally inferred from
the option strings. :class:`ArgumentParser` generates the value of ``dest`` by
taking the first long option string and stripping away the initial ``--``
string. If no long option strings were supplied, ``dest`` will be derived from
taking the first double-dash long option string and stripping away the initial
``-`` characters.
If no double-dash long option strings were supplied, ``dest`` will be derived
from the first single-dash long option string by stripping the initial ``-``
character.
If no long option strings were supplied, ``dest`` will be derived from
the first short option string by stripping the initial ``-`` character. Any
internal ``-`` characters will be converted to ``_`` characters to make sure
the string is a valid attribute name. The examples below illustrate this
behavior::

>>> parser = argparse.ArgumentParser()
>>> parser.add_argument('-f', '--foo-bar', '--foo')
>>> parser.add_argument('-q', '-quz')
>>> parser.add_argument('-x', '-y')
>>> parser.parse_args('-f 1 -x 2'.split())
Namespace(foo_bar='1', x='2')
>>> parser.parse_args('--foo 1 -y 2'.split())
Namespace(foo_bar='1', x='2')
>>> parser.parse_args('-f 1 -q 2 -x 3'.split())
Namespace(foo_bar='1', quz='2', x='3')
>>> parser.parse_args('--foo 1 -quz 2 -y 3'.split())
Namespace(foo_bar='1', quz='2', x='2')

``dest`` allows a custom attribute name to be provided::

Expand All @@ -1344,6 +1349,9 @@ behavior::
>>> parser.parse_args('--foo XXX'.split())
Namespace(bar='XXX')

.. versionchanged:: next
Single-dash long option now takes precedence over short options.


.. _deprecated:

Expand Down
7 changes: 7 additions & 0 deletions Doc/whatsnew/3.15.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1275,3 +1275,10 @@ that may require changes to your code.
Use its :meth:`!close` method or the :func:`contextlib.closing` context
manager to close it.
(Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.)

* If a short option and a single-dash long option are passed to
:meth:`argparse.ArgumentParser.add_argument`, *dest* is now inferred from
the single-dash long option. For example, in ``add_argument('-f', '-foo')``,
*dest* is now ``'foo'`` instead of ``'f'``.
Pass an explicit *dest* argument to preserve the old behavior.
(Contributed by Serhiy Storchaka in :gh:`138697`.)
1 change: 1 addition & 0 deletions Include/internal/pycore_global_objects_fini_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Include/internal/pycore_global_strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -793,6 +793,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(symmetric_difference_update)
STRUCT_FOR_ID(tabsize)
STRUCT_FOR_ID(tag)
STRUCT_FOR_ID(take_bytes)
STRUCT_FOR_ID(target)
STRUCT_FOR_ID(target_is_directory)
STRUCT_FOR_ID(task)
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_runtime_init_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions Include/internal/pycore_unicodeobject_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 17 additions & 11 deletions Lib/argparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -1660,29 +1660,35 @@ def _get_positional_kwargs(self, dest, **kwargs):
def _get_optional_kwargs(self, *args, **kwargs):
# determine short and long option strings
option_strings = []
long_option_strings = []
for option_string in args:
# error on strings that don't start with an appropriate prefix
if not option_string[0] in self.prefix_chars:
raise ValueError(
f'invalid option string {option_string!r}: '
f'must start with a character {self.prefix_chars!r}')

# strings starting with two prefix characters are long options
option_strings.append(option_string)
if len(option_string) > 1 and option_string[1] in self.prefix_chars:
long_option_strings.append(option_string)

# infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
dest = kwargs.pop('dest', None)
if dest is None:
if long_option_strings:
dest_option_string = long_option_strings[0]
else:
dest_option_string = option_strings[0]
dest = dest_option_string.lstrip(self.prefix_chars)
priority = 0
for option_string in option_strings:
if len(option_string) <= 2:
# short option: '-x' -> 'x'
if priority < 1:
dest = option_string.lstrip(self.prefix_chars)
priority = 1
elif option_string[1] not in self.prefix_chars:
# single-dash long option: '-foo' -> 'foo'
if priority < 2:
dest = option_string.lstrip(self.prefix_chars)
priority = 2
else:
# two-dash long option: '--foo' -> 'foo'
dest = option_string.lstrip(self.prefix_chars)
break
if not dest:
msg = f'dest= is required for options like {option_string!r}'
msg = f'dest= is required for options like {repr(option_strings)[1:-1]}'
raise TypeError(msg)
dest = dest.replace('-', '_')

Expand Down
19 changes: 9 additions & 10 deletions Lib/profiling/sampling/collector.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
from abc import ABC, abstractmethod

# Thread status flags
try:
from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED
except ImportError:
# Fallback for tests or when module is not available
THREAD_STATUS_HAS_GIL = (1 << 0)
THREAD_STATUS_ON_CPU = (1 << 1)
THREAD_STATUS_UNKNOWN = (1 << 2)
THREAD_STATUS_GIL_REQUESTED = (1 << 3)
from .constants import (
THREAD_STATUS_HAS_GIL,
THREAD_STATUS_ON_CPU,
THREAD_STATUS_UNKNOWN,
THREAD_STATUS_GIL_REQUESTED,
)

class Collector(ABC):
@abstractmethod
def collect(self, stack_frames):
"""Collect profiling data from stack frames."""

def collect_failed_sample(self):
"""Collect data about a failed sample attempt."""

@abstractmethod
def export(self, filename):
"""Export collected data to a file."""
Expand Down
30 changes: 30 additions & 0 deletions Lib/profiling/sampling/constants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
"""Constants for the sampling profiler."""

# Profiling mode constants
PROFILING_MODE_WALL = 0
PROFILING_MODE_CPU = 1
PROFILING_MODE_GIL = 2
PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks

# Sort mode constants
SORT_MODE_NSAMPLES = 0
SORT_MODE_TOTTIME = 1
SORT_MODE_CUMTIME = 2
SORT_MODE_SAMPLE_PCT = 3
SORT_MODE_CUMUL_PCT = 4
SORT_MODE_NSAMPLES_CUMUL = 5

# Thread status flags
try:
from _remote_debugging import (
THREAD_STATUS_HAS_GIL,
THREAD_STATUS_ON_CPU,
THREAD_STATUS_UNKNOWN,
THREAD_STATUS_GIL_REQUESTED,
)
except ImportError:
# Fallback for tests or when module is not available
THREAD_STATUS_HAS_GIL = (1 << 0)
THREAD_STATUS_ON_CPU = (1 << 1)
THREAD_STATUS_UNKNOWN = (1 << 2)
THREAD_STATUS_GIL_REQUESTED = (1 << 3)
200 changes: 200 additions & 0 deletions Lib/profiling/sampling/live_collector/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
"""Live profiling collector that displays top-like statistics using curses.

┌─────────────────────────────┐
│ Target Python Process │
│ (being profiled) │
└──────────────┬──────────────┘
│ Stack sampling at
│ configured interval
│ (e.g., 10000µs)
┌─────────────────────────────┐
│ LiveStatsCollector │
│ ┌───────────────────────┐ │
│ │ collect() │ │ Aggregates samples
│ │ - Iterates frames │ │ into statistics
│ │ - Updates counters │ │
│ └───────────┬───────────┘ │
│ │ │
│ ▼ │
│ ┌───────────────────────┐ │
│ │ Data Storage │ │
│ │ - result dict │ │ Tracks per-function:
│ │ - direct_calls │ │ • Direct samples
│ │ - cumulative_calls │ │ • Cumulative samples
│ └───────────┬───────────┘ │ • Derived time stats
│ │ │
│ ▼ │
│ ┌───────────────────────┐ │
│ │ Display Update │ │
│ │ (10Hz by default) │ │ Rate-limited refresh
│ └───────────┬───────────┘ │
└──────────────┼──────────────┘
┌─────────────────────────────┐
│ DisplayInterface │
│ (Abstract layer) │
└──────────────┬──────────────┘
┌───────┴────────┐
│ │
┌──────────▼────────┐ ┌───▼──────────┐
│ CursesDisplay │ │ MockDisplay │
│ - Real terminal │ │ - Testing │
│ - ncurses backend │ │ - No UI │
└─────────┬─────────┘ └──────────────┘
┌─────────────────────────────────────┐
│ Widget-Based Rendering │
│ ┌─────────────────────────────────┐ │
│ │ HeaderWidget │ │
│ │ • PID, uptime, time, interval │ │
│ │ • Sample stats & progress bar │ │
│ │ • Efficiency bar │ │
│ │ • Thread status & GC stats │ │
│ │ • Function summary │ │
│ │ • Top 3 hottest functions │ │
│ ├─────────────────────────────────┤ │
│ │ TableWidget │ │
│ │ • Column headers (sortable) │ │ Interactive display
│ │ • Stats rows (scrolling) │ │ with keyboard controls:
│ │ - nsamples % time │ │ s: sort, p: pause
│ │ - function file:line │ │ r: reset, /: filter
│ ├─────────────────────────────────┤ │ q: quit, h: help
│ │ FooterWidget │ │
│ │ • Legend and status │ │
│ │ • Filter input prompt │ │
│ └─────────────────────────────────┘ │
└─────────────────────────────────────┘

Architecture:

The live collector is organized into four layers. The data collection layer
(LiveStatsCollector) aggregates stack samples into per-function statistics without
any knowledge of how they will be presented. The display abstraction layer
(DisplayInterface) defines rendering operations without coupling to curses or any
specific UI framework. The widget layer (Widget, HeaderWidget, TableWidget,
FooterWidget, HelpWidget, ProgressBarWidget) encapsulates individual UI components
with their own rendering logic, promoting modularity and reusability. The
presentation layer (CursesDisplay/MockDisplay) implements the actual rendering for
terminal output and testing.

The system runs two independent update loops. The sampling loop is driven by the
profiler at the configured interval (e.g., 10000µs) and continuously collects
stack frames and updates statistics. The display loop runs at a fixed refresh rate
(default 10Hz) and updates the terminal independently of sampling frequency. This
separation allows high-frequency sampling without overwhelming the terminal with
constant redraws.

Statistics are computed incrementally as samples arrive. The collector maintains
running counters (direct calls and cumulative calls) in a dictionary keyed by
function location. Derived metrics like time estimates and percentages are computed
on-demand during display updates rather than being stored, which minimizes memory
overhead as the number of tracked functions grows.

User input is processed asynchronously during display updates using non-blocking I/O.
This allows interactive controls (sorting, filtering, pausing) without interrupting
the data collection pipeline. The collector maintains mode flags (paused,
filter_input_mode) that affect what gets displayed but not what gets collected.

"""

# Re-export all public classes and constants for backward compatibility
from .collector import LiveStatsCollector
from .display import DisplayInterface, CursesDisplay, MockDisplay
from .widgets import (
Widget,
ProgressBarWidget,
HeaderWidget,
TableWidget,
FooterWidget,
HelpWidget,
)
from .constants import (
MICROSECONDS_PER_SECOND,
DISPLAY_UPDATE_HZ,
DISPLAY_UPDATE_INTERVAL,
MIN_TERMINAL_WIDTH,
MIN_TERMINAL_HEIGHT,
WIDTH_THRESHOLD_SAMPLE_PCT,
WIDTH_THRESHOLD_TOTTIME,
WIDTH_THRESHOLD_CUMUL_PCT,
WIDTH_THRESHOLD_CUMTIME,
HEADER_LINES,
FOOTER_LINES,
SAFETY_MARGIN,
TOP_FUNCTIONS_DISPLAY_COUNT,
COL_WIDTH_NSAMPLES,
COL_SPACING,
COL_WIDTH_SAMPLE_PCT,
COL_WIDTH_TIME,
MIN_FUNC_NAME_WIDTH,
MAX_FUNC_NAME_WIDTH,
MIN_AVAILABLE_SPACE,
MIN_BAR_WIDTH,
MAX_SAMPLE_RATE_BAR_WIDTH,
MAX_EFFICIENCY_BAR_WIDTH,
MIN_SAMPLE_RATE_FOR_SCALING,
FINISHED_BANNER_EXTRA_LINES,
COLOR_PAIR_HEADER_BG,
COLOR_PAIR_CYAN,
COLOR_PAIR_YELLOW,
COLOR_PAIR_GREEN,
COLOR_PAIR_MAGENTA,
COLOR_PAIR_RED,
COLOR_PAIR_SORTED_HEADER,
DEFAULT_SORT_BY,
DEFAULT_DISPLAY_LIMIT,
)

__all__ = [
# Main collector
"LiveStatsCollector",
# Display interfaces
"DisplayInterface",
"CursesDisplay",
"MockDisplay",
# Widgets
"Widget",
"ProgressBarWidget",
"HeaderWidget",
"TableWidget",
"FooterWidget",
"HelpWidget",
# Constants
"MICROSECONDS_PER_SECOND",
"DISPLAY_UPDATE_HZ",
"DISPLAY_UPDATE_INTERVAL",
"MIN_TERMINAL_WIDTH",
"MIN_TERMINAL_HEIGHT",
"WIDTH_THRESHOLD_SAMPLE_PCT",
"WIDTH_THRESHOLD_TOTTIME",
"WIDTH_THRESHOLD_CUMUL_PCT",
"WIDTH_THRESHOLD_CUMTIME",
"HEADER_LINES",
"FOOTER_LINES",
"SAFETY_MARGIN",
"TOP_FUNCTIONS_DISPLAY_COUNT",
"COL_WIDTH_NSAMPLES",
"COL_SPACING",
"COL_WIDTH_SAMPLE_PCT",
"COL_WIDTH_TIME",
"MIN_FUNC_NAME_WIDTH",
"MAX_FUNC_NAME_WIDTH",
"MIN_AVAILABLE_SPACE",
"MIN_BAR_WIDTH",
"MAX_SAMPLE_RATE_BAR_WIDTH",
"MAX_EFFICIENCY_BAR_WIDTH",
"MIN_SAMPLE_RATE_FOR_SCALING",
"FINISHED_BANNER_EXTRA_LINES",
"COLOR_PAIR_HEADER_BG",
"COLOR_PAIR_CYAN",
"COLOR_PAIR_YELLOW",
"COLOR_PAIR_GREEN",
"COLOR_PAIR_MAGENTA",
"COLOR_PAIR_RED",
"COLOR_PAIR_SORTED_HEADER",
"DEFAULT_SORT_BY",
"DEFAULT_DISPLAY_LIMIT",
]
Loading
Loading