diff -Nru python3.7-3.7.0/aclocal.m4 python3.7-3.7.1/aclocal.m4 --- python3.7-3.7.0/aclocal.m4 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/aclocal.m4 2018-10-20 06:04:19.000000000 +0000 @@ -12,9 +12,9 @@ # PARTICULAR PURPOSE. m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) -# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -# serial 12 (pkg-config-0.29.2) - +dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +dnl serial 11 (pkg-config-0.29.1) +dnl dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson dnl @@ -55,7 +55,7 @@ dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29.2]) +[m4_define([PKG_MACROS_VERSION], [0.29.1]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ @@ -156,7 +156,7 @@ AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl pkg_failed=no -AC_MSG_CHECKING([for $2]) +AC_MSG_CHECKING([for $1]) _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) _PKG_CONFIG([$1][_LIBS], [libs], [$2]) @@ -166,11 +166,11 @@ See the pkg-config man page for more details.]) if test $pkg_failed = yes; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` - else + else $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi # Put the nasty error message in config.log where it belongs @@ -187,7 +187,7 @@ _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full diff -Nru python3.7-3.7.0/.azure-pipelines/ci.yml python3.7-3.7.1/.azure-pipelines/ci.yml --- python3.7-3.7.0/.azure-pipelines/ci.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/ci.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,136 @@ +variables: + manylinux: false + coverage: false + +jobs: +- job: Prebuild + displayName: Pre-build checks + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./prebuild-checks.yml + + +- job: Docs_PR + displayName: Docs PR + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./docs-steps.yml + parameters: + upload: true + + +- job: macOS_CI_Tests + displayName: macOS CI Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + variables: + testRunTitle: '$(build.sourceBranchName)-macos' + testRunPlatform: macos + + pool: + vmImage: xcode9-macos10.13 + + steps: + - template: ./macos-steps.yml + + +- job: Ubuntu_CI_Tests + displayName: Ubuntu CI Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(build.sourceBranchName)-linux' + testRunPlatform: linux + openssl_version: 1.1.0g + + steps: + - template: ./posix-steps.yml + + +- job: ManyLinux1_CI_Tests + displayName: ManyLinux1 CI Tests + dependsOn: Prebuild + condition: | + and( + and( + succeeded(), + eq(variables['manylinux'], 'true') + ), + eq(dependencies.Prebuild.outputs['tests.run'], 'true') + ) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(build.sourceBranchName)-manylinux1' + testRunPlatform: manylinux1 + imageName: 'dockcross/manylinux-x64' + + steps: + - template: ./docker-steps.yml + + +- job: Ubuntu_Coverage_CI_Tests + displayName: Ubuntu CI Tests (coverage) + dependsOn: Prebuild + condition: | + and( + and( + succeeded(), + eq(variables['coverage'], 'true') + ), + eq(dependencies.Prebuild.outputs['tests.run'], 'true') + ) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(Build.SourceBranchName)-linux-coverage' + testRunPlatform: linux-coverage + openssl_version: 1.1.0g + + steps: + - template: ./posix-steps.yml + parameters: + coverage: true + + +- job: Windows_CI_Tests + displayName: Windows CI Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: vs2017-win2016 + + strategy: + matrix: + win32: + arch: win32 + buildOpt: + testRunTitle: '$(Build.SourceBranchName)-win32' + testRunPlatform: win32 + win64: + arch: amd64 + buildOpt: '-p x64' + testRunTitle: '$(Build.SourceBranchName)-win64' + testRunPlatform: win64 + maxParallel: 2 + + steps: + - template: ./windows-steps.yml diff -Nru python3.7-3.7.0/.azure-pipelines/docker-steps.yml python3.7-3.7.1/.azure-pipelines/docker-steps.yml --- python3.7-3.7.0/.azure-pipelines/docker-steps.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/docker-steps.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,76 @@ +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- ${{ if ne(parameters.targetBranch, '') }}: + - script: | + git fetch -q origin ${{ parameters.targetbranch }} + if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' + then + echo "Only docs were updated, stopping build process." + echo "##vso[task.setvariable variable=DocOnly]true" + exit + fi + displayName: Detect doc-only changes + +- task: docker@0 + displayName: 'Configure CPython (debug)' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: './configure --with-pydebug' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: docker@0 + displayName: 'Build CPython' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: 'make -s -j4' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: docker@0 + displayName: 'Display build info' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: 'make pythoninfo' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: docker@0 + displayName: 'Tests' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '$(build.binariesDirectory)/test-results.xml' + mergeTestResults: true + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) + condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true')) diff -Nru python3.7-3.7.0/.azure-pipelines/docs-steps.yml python3.7-3.7.1/.azure-pipelines/docs-steps.yml --- python3.7-3.7.0/.azure-pipelines/docs-steps.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/docs-steps.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,46 @@ +parameters: + latex: false + upload: false + +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + +- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme + displayName: 'Install build dependencies' + +- ${{ if ne(parameters.latex, 'true') }}: + - script: make check suspicious html PYTHON=python + workingDirectory: '$(build.sourcesDirectory)/Doc' + displayName: 'Build documentation' + +- ${{ if eq(parameters.latex, 'true') }}: + - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full + displayName: 'Install LaTeX' + + - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' + workingDirectory: '$(build.sourcesDirectory)/Doc' + displayName: 'Build documentation' + +- ${{ if eq(parameters.upload, 'true') }}: + - task: PublishBuildArtifacts@1 + displayName: 'Publish docs' + + inputs: + PathToPublish: '$(build.sourcesDirectory)/Doc/build' + ArtifactName: docs + publishLocation: Container + + - ${{ if eq(parameters.latex, 'true') }}: + - task: PublishBuildArtifacts@1 + displayName: 'Publish dist' + inputs: + PathToPublish: '$(build.sourcesDirectory)/Doc/dist' + ArtifactName: docs_dist + publishLocation: Container diff -Nru python3.7-3.7.0/.azure-pipelines/macos-steps.yml python3.7-3.7.1/.azure-pipelines/macos-steps.yml --- python3.7-3.7.0/.azure-pipelines/macos-steps.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/macos-steps.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,25 @@ +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev + displayName: 'Configure CPython (debug)' + +- script: make -s -j4 + displayName: 'Build CPython' + +- script: make pythoninfo + displayName: 'Display build info' + +- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" + displayName: 'Tests' + +- task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '$(build.binariesDirectory)/test-results.xml' + mergeTestResults: true + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) + condition: succeededOrFailed() diff -Nru python3.7-3.7.0/.azure-pipelines/posix-deps.sh python3.7-3.7.1/.azure-pipelines/posix-deps.sh --- python3.7-3.7.0/.azure-pipelines/posix-deps.sh 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/posix-deps.sh 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,26 @@ +sudo apt-get update + +sudo apt-get -yq install \ + build-essential \ + zlib1g-dev \ + libbz2-dev \ + liblzma-dev \ + libncurses5-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + libgdbm-dev \ + tk-dev \ + lzma \ + lzma-dev \ + liblzma-dev \ + libffi-dev \ + uuid-dev \ + xvfb + +if [ ! -z "$1" ] +then + echo ##vso[task.prependpath]$PWD/multissl/openssl/$1 + echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1 + python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux +fi diff -Nru python3.7-3.7.0/.azure-pipelines/posix-steps.yml python3.7-3.7.1/.azure-pipelines/posix-steps.yml --- python3.7-3.7.0/.azure-pipelines/posix-steps.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/posix-steps.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,63 @@ +parameters: + coverage: false + +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- script: ./.azure-pipelines/posix-deps.sh $(openssl_version) + displayName: 'Install dependencies' + +- script: ./configure --with-pydebug + displayName: 'Configure CPython (debug)' + +- script: make -s -j4 + displayName: 'Build CPython' + +- ${{ if eq(parameters.coverage, 'true') }}: + - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage + displayName: 'Set up virtual environment' + + - script: ./venv/bin/python -m test.pythoninfo + displayName: 'Display build info' + + - script: | + xvfb-run ./venv/bin/python -m coverage run --pylib -m test \ + --fail-env-changed \ + -uall,-cpu \ + --junit-xml=$(build.binariesDirectory)/test-results.xml" \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures + displayName: 'Tests with coverage' + + - script: ./venv/bin/python -m coverage xml + displayName: 'Generate coverage.xml' + + - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) + displayName: 'Publish code coverage results' + + +- ${{ if ne(parameters.coverage, 'true') }}: + - script: make pythoninfo + displayName: 'Display build info' + + - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" + displayName: 'Tests' + + +- script: ./python Tools/scripts/patchcheck.py --travis true + displayName: 'Run patchcheck.py' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + + +- task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '$(build.binariesDirectory)/test-results.xml' + mergeTestResults: true + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) + condition: succeededOrFailed() diff -Nru python3.7-3.7.0/.azure-pipelines/prebuild-checks.yml python3.7-3.7.1/.azure-pipelines/prebuild-checks.yml --- python3.7-3.7.0/.azure-pipelines/prebuild-checks.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/prebuild-checks.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,36 @@ +steps: +- checkout: self + fetchDepth: 5 + +- script: echo "##vso[task.setvariable variable=diffTarget]HEAD~1" + displayName: Set default diff target + +- script: | + git fetch -q origin $(System.PullRequest.TargetBranch) + echo "##vso[task.setvariable variable=diffTarget]HEAD \$(git merge-base HEAD FETCH_HEAD)" + displayName: Fetch comparison tree + condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) + +- script: | + if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)' + then + echo "No docs were updated: docs.run=false" + echo "##vso[task.setvariable variable=run;isOutput=true]false" + else + echo "Docs were updated: docs.run=true" + echo "##vso[task.setvariable variable=run;isOutput=true]true" + fi + displayName: Detect documentation changes + name: docs + +- script: | + if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)' + then + echo "Only docs were updated: tests.run=false" + echo "##vso[task.setvariable variable=run;isOutput=true]false" + else + echo "Code was updated: tests.run=true" + echo "##vso[task.setvariable variable=run;isOutput=true]true" + fi + displayName: Detect source changes + name: tests diff -Nru python3.7-3.7.0/.azure-pipelines/pr.yml python3.7-3.7.1/.azure-pipelines/pr.yml --- python3.7-3.7.0/.azure-pipelines/pr.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/pr.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,86 @@ +jobs: +- job: Prebuild + displayName: Pre-build checks + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./prebuild-checks.yml + + +- job: Docs_PR + displayName: Docs PR + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./docs-steps.yml + + +- job: macOS_PR_Tests + displayName: macOS PR Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + variables: + testRunTitle: '$(system.pullRequest.TargetBranch)-macos' + testRunPlatform: macos + + pool: + vmImage: xcode9-macos10.13 + + steps: + - template: ./macos-steps.yml + parameters: + targetBranch: $(System.PullRequest.TargetBranch) + + +- job: Ubuntu_PR_Tests + displayName: Ubuntu PR Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(system.pullRequest.TargetBranch)-linux' + testRunPlatform: linux + openssl_version: 1.1.0g + + steps: + - template: ./posix-steps.yml + parameters: + targetBranch: $(System.PullRequest.TargetBranch) + + +- job: Windows_PR_Tests + displayName: Windows PR Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: vs2017-win2016 + + strategy: + matrix: + win32: + arch: win32 + buildOpt: + testRunTitle: '$(System.PullRequest.TargetBranch)-win32' + testRunPlatform: win32 + win64: + arch: amd64 + buildOpt: '-p x64' + testRunTitle: '$(System.PullRequest.TargetBranch)-win64' + testRunPlatform: win64 + maxParallel: 2 + + steps: + - template: ./windows-steps.yml + parameters: + targetBranch: $(System.PullRequest.TargetBranch) diff -Nru python3.7-3.7.0/.azure-pipelines/windows-steps.yml python3.7-3.7.1/.azure-pipelines/windows-steps.yml --- python3.7-3.7.0/.azure-pipelines/windows-steps.yml 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/.azure-pipelines/windows-steps.yml 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,32 @@ +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- powershell: | + # Relocate build outputs outside of source directory to make cleaning faster + Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj' + # UNDONE: Do not build to a different directory because of broken tests + Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild' + Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals' + displayName: Update build locations + +- script: PCbuild\build.bat -e $(buildOpt) + displayName: 'Build CPython' + +- script: python.bat -m test.pythoninfo + displayName: 'Display build info' + +- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" + displayName: 'Tests' + env: + PREFIX: $(Py_OutDir)\$(arch) + +- task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml' + mergeTestResults: true + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) + condition: succeededOrFailed() diff -Nru python3.7-3.7.0/CODE_OF_CONDUCT.rst python3.7-3.7.1/CODE_OF_CONDUCT.rst --- python3.7-3.7.0/CODE_OF_CONDUCT.rst 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/CODE_OF_CONDUCT.rst 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,14 @@ +Code of Conduct +=============== + +Please note that all interactions on +`Python Software Foundation `__-supported +infrastructure is `covered +`__ +by the `PSF Code of Conduct `__, +which includes all infrastructure used in the development of Python itself +(e.g. mailing lists, issue trackers, GitHub, etc.). + +In general this means everyone is expected to be open, considerate, and +respectful of others no matter what their position is within the project. + diff -Nru python3.7-3.7.0/configure python3.7-3.7.1/configure --- python3.7-3.7.0/configure 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/configure 2018-10-20 06:04:19.000000000 +0000 @@ -831,7 +831,6 @@ with_doc_strings with_pymalloc with_c_locale_coercion -with_c_locale_warning with_valgrind with_dtrace with_libm @@ -1527,9 +1526,6 @@ --with(out)-c-locale-coercion disable/enable C locale coercion to a UTF-8 based locale - --with(out)-c-locale-warning - disable/enable locale compatibility warning in the C - locale --with-valgrind Enable Valgrind support --with(out)-dtrace disable/enable DTrace support --with-libm=STRING math library @@ -2753,9 +2749,9 @@ fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" - GITTAG="git -C \$(srcdir) describe --all --always --dirty" - GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" + GITVERSION="git --git-dir \$(srcdir)/.git rev-parse --short HEAD" + GITTAG="git --git-dir \$(srcdir)/.git describe --all --always --dirty" + GITBRANCH="git --git-dir \$(srcdir)/.git name-rev --name-only HEAD" else GITVERSION="" GITTAG="" @@ -6482,6 +6478,14 @@ esac ;; esac + + if test "$ac_cv_prog_cc_g" = "yes" + then + # bpo-30345: Add -g to LDFLAGS when compiling with LTO + # to get debug symbols. + LTOFLAGS="$LTOFLAGS -g" + fi + CFLAGS="$CFLAGS $LTOFLAGS" LDFLAGS="$LDFLAGS $LTOFLAGS" fi @@ -11120,29 +11124,6 @@ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_c_locale_coercion" >&5 $as_echo "$with_c_locale_coercion" >&6; } -# Check for --with-c-locale-warning -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-c-locale-warning" >&5 -$as_echo_n "checking for --with-c-locale-warning... " >&6; } - -# Check whether --with-c-locale-warning was given. -if test "${with_c_locale_warning+set}" = set; then : - withval=$with_c_locale_warning; -fi - - -if test -z "$with_c_locale_warning" -then - with_c_locale_warning="yes" -fi -if test "$with_c_locale_warning" != "no" -then - -$as_echo "#define PY_WARN_ON_C_LOCALE 1" >>confdefs.h - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_c_locale_warning" >&5 -$as_echo "$with_c_locale_warning" >&6; } - # Check for Valgrind support { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-valgrind" >&5 $as_echo_n "checking for --with-valgrind... " >&6; } @@ -11346,7 +11327,7 @@ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ if_nameindex \ - initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ + initgroups kill killpg lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise posix_spawn pread preadv preadv2 \ @@ -11373,6 +11354,23 @@ done +# Force lchmod off for Linux. Linux disallows changing the mode of symbolic +# links. Some libc implementations have a stub lchmod implementation that always +# returns an error. +if test "$MACHDEP" != linux; then + for ac_func in lchmod +do : + ac_fn_c_check_func "$LINENO" "lchmod" "ac_cv_func_lchmod" +if test "x$ac_cv_func_lchmod" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LCHMOD 1 +_ACEOF + +fi +done + +fi + ac_fn_c_check_decl "$LINENO" "dirfd" "ac_cv_have_decl_dirfd" "#include #include " @@ -14701,19 +14699,21 @@ $as_echo "$ac_cv_wchar_t_signed" >&6; } fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether wchar_t is usable" >&5 +$as_echo_n "checking whether wchar_t is usable... " >&6; } # wchar_t is only usable if it maps to an unsigned type if test "$ac_cv_sizeof_wchar_t" -ge 2 \ -a "$ac_cv_wchar_t_signed" = "no" then - HAVE_USABLE_WCHAR_T="yes" $as_echo "#define HAVE_USABLE_WCHAR_T 1" >>confdefs.h + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } else - HAVE_USABLE_WCHAR_T="no usable wchar_t found" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAVE_USABLE_WCHAR_T" >&5 -$as_echo "$HAVE_USABLE_WCHAR_T" >&6; } # check for endianness { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether byte ordering is bigendian" >&5 @@ -15510,77 +15510,6 @@ fi -# Before we can test tzset, we need to check if struct tm has a tm_zone -# (which is not required by ISO C or UNIX spec) and/or if we support -# tzname[] -ac_fn_c_check_member "$LINENO" "struct tm" "tm_zone" "ac_cv_member_struct_tm_tm_zone" "#include -#include <$ac_cv_struct_tm> - -" -if test "x$ac_cv_member_struct_tm_tm_zone" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_STRUCT_TM_TM_ZONE 1 -_ACEOF - - -fi - -if test "$ac_cv_member_struct_tm_tm_zone" = yes; then - -$as_echo "#define HAVE_TM_ZONE 1" >>confdefs.h - -else - ac_fn_c_check_decl "$LINENO" "tzname" "ac_cv_have_decl_tzname" "#include -" -if test "x$ac_cv_have_decl_tzname" = xyes; then : - ac_have_decl=1 -else - ac_have_decl=0 -fi - -cat >>confdefs.h <<_ACEOF -#define HAVE_DECL_TZNAME $ac_have_decl -_ACEOF - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for tzname" >&5 -$as_echo_n "checking for tzname... " >&6; } -if ${ac_cv_var_tzname+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -#if !HAVE_DECL_TZNAME -extern char *tzname[]; -#endif - -int -main () -{ -return tzname[0][0]; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_var_tzname=yes -else - ac_cv_var_tzname=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_var_tzname" >&5 -$as_echo "$ac_cv_var_tzname" >&6; } - if test $ac_cv_var_tzname = yes; then - -$as_echo "#define HAVE_TZNAME 1" >>confdefs.h - - fi -fi - - # check tzset(3) exists and works like we expect it to { $as_echo "$as_me:${as_lineno-$LINENO}: checking for working tzset()" >&5 $as_echo_n "checking for working tzset()... " >&6; } @@ -16635,7 +16564,6 @@ #include atomic_int value = ATOMIC_VAR_INIT(1); - _Atomic void *py_atomic_address = (void*) &value; int main() { int loaded_value = atomic_load(&value); return 0; diff -Nru python3.7-3.7.0/configure.ac python3.7-3.7.1/configure.ac --- python3.7-3.7.0/configure.ac 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/configure.ac 2018-10-20 06:04:19.000000000 +0000 @@ -39,9 +39,9 @@ fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" - GITTAG="git -C \$(srcdir) describe --all --always --dirty" - GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" + GITVERSION="git --git-dir \$(srcdir)/.git rev-parse --short HEAD" + GITTAG="git --git-dir \$(srcdir)/.git describe --all --always --dirty" + GITBRANCH="git --git-dir \$(srcdir)/.git name-rev --name-only HEAD" else GITVERSION="" GITTAG="" @@ -1339,6 +1339,14 @@ esac ;; esac + + if test "$ac_cv_prog_cc_g" = "yes" + then + # bpo-30345: Add -g to LDFLAGS when compiling with LTO + # to get debug symbols. + LTOFLAGS="$LTOFLAGS -g" + fi + CFLAGS="$CFLAGS $LTOFLAGS" LDFLAGS="$LDFLAGS $LTOFLAGS" fi @@ -3377,23 +3385,6 @@ fi AC_MSG_RESULT($with_c_locale_coercion) -# Check for --with-c-locale-warning -AC_MSG_CHECKING(for --with-c-locale-warning) -AC_ARG_WITH(c-locale-warning, - AS_HELP_STRING([--with(out)-c-locale-warning], - [disable/enable locale compatibility warning in the C locale])) - -if test -z "$with_c_locale_warning" -then - with_c_locale_warning="yes" -fi -if test "$with_c_locale_warning" != "no" -then - AC_DEFINE(PY_WARN_ON_C_LOCALE, 1, - [Define to emit a locale compatibility warning in the C locale]) -fi -AC_MSG_RESULT($with_c_locale_warning) - # Check for Valgrind support AC_MSG_CHECKING([for --with-valgrind]) AC_ARG_WITH([valgrind], @@ -3512,7 +3503,7 @@ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ if_nameindex \ - initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ + initgroups kill killpg lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise posix_spawn pread preadv preadv2 \ @@ -3528,6 +3519,13 @@ truncate uname unlinkat unsetenv utimensat utimes waitid waitpid wait3 wait4 \ wcscoll wcsftime wcsxfrm wmemcmp writev _getpty) +# Force lchmod off for Linux. Linux disallows changing the mode of symbolic +# links. Some libc implementations have a stub lchmod implementation that always +# returns an error. +if test "$MACHDEP" != linux; then + AC_CHECK_FUNCS(lchmod) +fi + AC_CHECK_DECL(dirfd, AC_DEFINE(HAVE_DIRFD, 1, Define if you have the 'dirfd' function or macro.), , @@ -4630,19 +4628,19 @@ AC_MSG_RESULT($ac_cv_wchar_t_signed) fi +AC_MSG_CHECKING(whether wchar_t is usable) # wchar_t is only usable if it maps to an unsigned type if test "$ac_cv_sizeof_wchar_t" -ge 2 \ -a "$ac_cv_wchar_t_signed" = "no" then - HAVE_USABLE_WCHAR_T="yes" AC_DEFINE(HAVE_USABLE_WCHAR_T, 1, [Define if you have a useable wchar_t type defined in wchar.h; useable means wchar_t must be an unsigned type with at least 16 bits. (see Include/unicodeobject.h).]) + AC_MSG_RESULT(yes) else - HAVE_USABLE_WCHAR_T="no usable wchar_t found" + AC_MSG_RESULT(no) fi -AC_MSG_RESULT($HAVE_USABLE_WCHAR_T) # check for endianness AC_C_BIGENDIAN @@ -4868,11 +4866,6 @@ [Define if poll() sets errno on invalid file descriptors.]) fi -# Before we can test tzset, we need to check if struct tm has a tm_zone -# (which is not required by ISO C or UNIX spec) and/or if we support -# tzname[] -AC_STRUCT_TIMEZONE - # check tzset(3) exists and works like we expect it to AC_MSG_CHECKING(for working tzset()) AC_CACHE_VAL(ac_cv_working_tzset, [ @@ -5422,7 +5415,6 @@ AC_LANG_SOURCE([[ #include atomic_int value = ATOMIC_VAR_INIT(1); - _Atomic void *py_atomic_address = (void*) &value; int main() { int loaded_value = atomic_load(&value); return 0; @@ -5434,7 +5426,7 @@ if test "$have_stdatomic_h" = yes; then AC_DEFINE(HAVE_STD_ATOMIC, 1, - [Has stdatomic.h, atomic_int and _Atomic void* types work]) + [Has stdatomic.h with atomic_int]) fi # Check for GCC >= 4.7 __atomic builtins diff -Nru python3.7-3.7.0/debian/changelog python3.7-3.7.1/debian/changelog --- python3.7-3.7.0/debian/changelog 2018-09-12 18:30:08.000000000 +0000 +++ python3.7-3.7.1/debian/changelog 2018-10-22 11:21:55.000000000 +0000 @@ -1,3 +1,156 @@ +python3.7 (3.7.1-1~18.04) bionic-proposed; urgency=medium + + * SRU: LP: #1799202. + + -- Matthias Klose Mon, 22 Oct 2018 13:21:55 +0200 + +python3.7 (3.7.1-1) unstable; urgency=medium + + * Python 3.7.1 release. + + -- Matthias Klose Sun, 21 Oct 2018 10:03:53 +0200 + +python3.7 (3.7.1~rc2-1) unstable; urgency=medium + + * Python 3.7.1 release candidate 2. + + -- Matthias Klose Mon, 15 Oct 2018 11:24:12 +0200 + +python3.7 (3.7.1~rc1-1) unstable; urgency=medium + + * Python 3.7.1 release candidate 1. + + -- Matthias Klose Thu, 27 Sep 2018 11:41:42 +0200 + +python3.7 (3.7.0-7) unstable; urgency=medium + + * Update to 20180925 from the 3.7 branch. + * Don't run OpenSSL related tests during the build. Hanging on + some buildds. + + -- Matthias Klose Tue, 25 Sep 2018 16:01:24 +0200 + +python3.7 (3.7.0-6) unstable; urgency=medium + + * Update to 20180901 from the 3.7 branch. + - Fixes the build with OpenSSL 1.1.1. Closes: #907031. + * Don't run the test_ftplib and test_ssl autopkg tests, not yet ready + for OpenSSL 1.1.1. + + -- Matthias Klose Sat, 01 Sep 2018 01:21:37 +0200 + +python3.7 (3.7.0-5) unstable; urgency=medium + + * Update to 20180820 from the 3.7 branch. + * Avoid usage of distutils in the platform module. See issue #26544. + * Fix some issues on architectures from the computer museum (Jason Duerstock). + Closes: #905536. + - Disable -O3 on ia64 (see PR rtl-optimization/85412). + - Remove the -O2 override on m68k (see Debian #326903). + - Remove the -mieee and -O2 overrides for DEC Alpha (see Debian #212912). + * Add more breaks for packages not compatible with Python 3.7: + - python3-aiomeasures. See #906159. + - python3-motor (updated). See #903527. + * Again close the RC issue about the syntax errors triggered by Python 3.7. + Closes: #902788. + * Update symbols files. + + -- Matthias Klose Mon, 20 Aug 2018 19:52:20 +0200 + +python3.7 (3.7.0-4) unstable; urgency=medium + + * Update to 20180802 from the 3.7 branch. + * Add more breaks for packages not compatible with Python 3.7: + - python3-pyatspi. See #902989. + * Move test/__main__.py into libpython3.7-stdlib. + + -- Matthias Klose Thu, 02 Aug 2018 08:46:56 +0200 + +python3.7 (3.7.0-3) unstable; urgency=medium + + * Update to 20180730 from the 3.7 branch. + * Add more breaks for packages not compatible with Python 3.7: + - python3-trollius. See #903888. + - python3-sleekxmpp. See #904453. + - voltron. See #904657. + - visidata. See #904656. + - salt-common. See #903654. + - python3-yowsup. See #904653. + - python3-websockets. See #904650. + - python3-txfixtures. See #904649. + - python3-tweepy. See #904648. + - python3-slixmpp. See #904644. + - python3-signaller. See #904642. + - python3-rpyc. See #904615. + - python3-protorpc-standalone. See #904603. + - python3-nova. See #904587. + - python3-murano. See #904581. + - python3-mastodon. See #904579. + - python3-hug. See #904576. + - python3-async. See #904492. + - python3-pylama. See #904491. + - python3-exabgp. See #904390. + - python3-glance. See #904389. + - python3-gbulb. See #904388. + - python3-engineio. See #904383. + - python3-doublex. See #904382. + - python3-dns. See #904381. + - python3-opcua. See #904373. + - openscap-daemon. See #904. + - oca-core. See #904370. + - patroni. See #904369. + - python3-panoramisk. See #904368. + - linux-show-player-. See #904367. + - python3-cs. See #904365. + - python3-azure. See #904364. + - python3-applicationinsights. See #904363. + - python3-aiopg. See #904361. + - python3-aiozmq. See #904358. + - python3-pysnmp4. See #904357. + - python3-astroquery. See #904351. + - snakemake. See #904350. + - fail2ban. See #902817. + - python3-morse-simulator. See #904343. + - python3-tango. See #904298. + - python3-pycuda. See #903826. + - python3-pycsw. See #903784. + - gnome3-builder. See #903558. + * Close the RC issue about the syntax errors triggered by Python 3.7, now + that all the breaks are in place. Closes: #902788. + * Mark the lib2to3 and site tests as succeeding again, mark the tcl test + as failing (issue #34178). + + -- Matthias Klose Mon, 30 Jul 2018 06:39:20 +0200 + +python3.7 (3.7.0-2) unstable; urgency=medium + + * Update to 20180720 from the 3.7 branch. + * Add breaks for packages not compatible with Python 3.7: + - python3-twisted. See #902766. + - python3-pexpect. See #902646. + - python3-cxx-dev. See #902673. + - libboost-python1.62.0. See #902702. + - python3-psycopg2. See #902715. + - python3-maxminddb. See #902756. + - python3-blist. See #902757. + - python3-django. See #902761. + - diffoscope. See #902650. + - gpodder. See #902704. + - python3-http-parser. See: #902759. + - python3-celery. See #902900. + - python3-astroid. See #902631. + - python3-ubjson. See #902762. + - python3-protobuf. See #902597. + - python3-opengl. See #903218. + - python3-ws4py. See #903529. + - python3-dropbox. See #903525. + - python3-pycuda. See #903826. + - python3-libcloud. See #903388. + - python3-motor. See #903527. + - python3-jira. See #903526. + + -- Matthias Klose Fri, 20 Jul 2018 15:51:14 +0200 + python3.7 (3.7.0-1~18.04) bionic-proposed; urgency=medium * SRU: LP: #1792140. Update Python 3.7 to the final 3.7.0 release. diff -Nru python3.7-3.7.0/debian/control python3.7-3.7.1/debian/control --- python3.7-3.7.0/debian/control 2018-05-29 14:01:26.000000000 +0000 +++ python3.7-3.7.1/debian/control 2018-09-27 09:43:14.000000000 +0000 @@ -17,7 +17,7 @@ mime-support, netbase, bzip2, time, python3:any, net-tools, xvfb, xauth Build-Depends-Indep: python3-sphinx, texinfo -Standards-Version: 4.1.4 +Standards-Version: 4.2.1 Vcs-Browser: https://code.launchpad.net/~doko/python/pkg3.7-debian Vcs-Bzr: http://bazaar.launchpad.net/~doko/python/pkg3.7-debian XS-Testsuite: autopkgtest @@ -33,7 +33,7 @@ Suggests: python3.7-venv, python3.7-doc, binutils Breaks: python3-all (<< 3.6.5~rc1-1), python3-dev (<< 3.6.5~rc1-1), - python3-venv (<< 3.6.5-2) + python3-venv (<< 3.6.5-2), ${python37:Breaks} Description: Interactive high-level object-oriented language (version 3.7) Python is a high-level, interactive, object-oriented language. Its 3.7 version includes an extensive class library with lots of goodies for @@ -61,7 +61,8 @@ Depends: libpython3.7-minimal (= ${binary:Version}), mime-support, ${shlibs:Depends}, ${misc:Depends} Breaks: python3-tk (<< 3.6.4~rc1-2), libmpdec2 (<< 2.4.2), python3-distutils (<< 3.6.5~rc1-2), -Replaces: python3-distutils (<< 3.6.5~rc1-2), + libpython3.7-testsuite (<< 3.7.0-4), +Replaces: python3-distutils (<< 3.6.5~rc1-2), libpython3.7-testsuite (<< 3.7.0-4), Description: Interactive high-level object-oriented language (standard library, version 3.7) Python is a high-level, interactive, object-oriented language. Its 3.7 version includes an extensive class library with lots of goodies for diff -Nru python3.7-3.7.0/debian/control.in python3.7-3.7.1/debian/control.in --- python3.7-3.7.0/debian/control.in 2018-05-11 15:18:34.000000000 +0000 +++ python3.7-3.7.1/debian/control.in 2018-09-01 01:23:30.000000000 +0000 @@ -17,7 +17,7 @@ mime-support, netbase, bzip2, time, python3@bd_qual@, net-tools, xvfb, xauth Build-Depends-Indep: python3-sphinx, texinfo -Standards-Version: 4.1.4 +Standards-Version: 4.2.1 Vcs-Browser: https://code.launchpad.net/~doko/python/pkg@VER@-debian Vcs-Bzr: http://bazaar.launchpad.net/~doko/python/pkg@VER@-debian XS-Testsuite: autopkgtest @@ -33,7 +33,7 @@ Suggests: @PVER@-venv, @PVER@-doc, binutils Breaks: python3-all (<< 3.6.5~rc1-1), python3-dev (<< 3.6.5~rc1-1), - python3-venv (<< 3.6.5-2) + python3-venv (<< 3.6.5-2), ${python37:Breaks} Description: Interactive high-level object-oriented language (version @VER@) Python is a high-level, interactive, object-oriented language. Its @VER@ version includes an extensive class library with lots of goodies for @@ -61,7 +61,8 @@ Depends: lib@PVER@-minimal (= ${binary:Version}), mime-support, ${shlibs:Depends}, ${misc:Depends} Breaks: python3-tk (<< 3.6.4~rc1-2), libmpdec2 (<< 2.4.2), python3-distutils (<< 3.6.5~rc1-2), -Replaces: python3-distutils (<< 3.6.5~rc1-2), + libpython3.7-testsuite (<< 3.7.0-4), +Replaces: python3-distutils (<< 3.6.5~rc1-2), libpython3.7-testsuite (<< 3.7.0-4), Description: Interactive high-level object-oriented language (standard library, version @VER@) Python is a high-level, interactive, object-oriented language. Its @VER@ version includes an extensive class library with lots of goodies for diff -Nru python3.7-3.7.0/debian/libpython.symbols.in python3.7-3.7.1/debian/libpython.symbols.in --- python3.7-3.7.0/debian/libpython.symbols.in 2018-05-16 21:58:57.000000000 +0000 +++ python3.7-3.7.1/debian/libpython.symbols.in 2018-08-20 19:21:06.000000000 +0000 @@ -1181,7 +1181,9 @@ _PyContext_NewHamtForTests@Base @SVER@ _PyCoreConfig_Clear@Base @SVER@ _PyCoreConfig_Copy@Base @SVER@ + _PyCoreConfig_GetGlobalConfig@Base @SVER@ _PyCoreConfig_Read@Base @SVER@ + _PyCoreConfig_SetGlobalConfig@Base @SVER@ _PyCoroWrapper_Type@Base @SVER@ _PyCoro_GetAwaitableIter@Base @SVER@ _PyDebugAllocatorStats@Base @SVER@ @@ -1715,8 +1717,10 @@ _Py_ImportFrom@Base @SVER@ _Py_Index@Base @SVER@ _Py_InitializeCore@Base @SVER@ - _Py_InitializeEx_Private@Base @SVER@ + _Py_InitializeCore_impl@Base @SVER@ + _Py_InitializeFromConfig@Base @SVER@ _Py_InitializeMainInterpreter@Base @SVER@ + _Py_Initialize_ReadEnvVarsNoAlloc@Base @SVER@ _Py_Interactive@Base @SVER@ _Py_IsCoreInitialized@Base @SVER@ _Py_IsFinalizing@Base @SVER@ diff -Nru python3.7-3.7.0/debian/patches/git-updates.diff python3.7-3.7.1/debian/patches/git-updates.diff --- python3.7-3.7.0/debian/patches/git-updates.diff 2018-06-27 14:41:15.000000000 +0000 +++ python3.7-3.7.1/debian/patches/git-updates.diff 2018-09-27 09:42:50.000000000 +0000 @@ -1,4 +1,4 @@ -# DP: updates from the 3.7 branch (until 2018-06-22). +# DP: updates from the 3.7 branch (until 2018-09-26). -# git diff --no-renames dfad352267cee3ea581035622a7371bc3b235378 53d1e9fad316e1404535157fe21cab8919f707c9 | filterdiff -x ?/.hgignore -x ?/.hgeol -x ?/.hgtags -x ?/.hgtouch -x ?/.gitignore -x ?/.gitattributes -x '?/.github/*' -x '?/.git*' -x ?/.codecov.yml -x ?/.travis.yml -x ?/configure --remove-timestamps +# git diff --no-renames 1bf9cc509326bc42cd8cb1650eb9bf64550d817e 69d63bbbd6ea8fab7792a7e4de21a4662665cb5f | filterdiff -x ?/.hgignore -x ?/.hgeol -x ?/.hgtags -x ?/.hgtouch -x ?/.gitignore -x ?/.gitattributes -x '?/.github/*' -x '?/.git*' -x ?/.codecov.yml -x ?/.travis.yml -x ?/configure --remove-timestamps diff -Nru python3.7-3.7.0/debian/patches/series python3.7-3.7.1/debian/patches/series --- python3.7-3.7.0/debian/patches/series 2018-06-27 14:41:22.000000000 +0000 +++ python3.7-3.7.1/debian/patches/series 2018-09-27 09:42:57.000000000 +0000 @@ -1,4 +1,4 @@ -# git-updates.diff +#git-updates.diff deb-setup.diff deb-locations.diff distutils-install-layout.diff diff -Nru python3.7-3.7.0/debian/patches/setup-modules.diff python3.7-3.7.1/debian/patches/setup-modules.diff --- python3.7-3.7.0/debian/patches/setup-modules.diff 2018-04-28 12:06:15.000000000 +0000 +++ python3.7-3.7.1/debian/patches/setup-modules.diff 2018-09-25 13:54:41.000000000 +0000 @@ -41,8 +41,8 @@ # Andrew Kuchling's zlib module. # This require zlib 1.1.3 (or later). -@@ -348,7 +347,7 @@ _symtable symtablemodule.c - # +@@ -340,7 +339,7 @@ _symtable symtablemodule.c + # Interface to the Expat XML parser # More information on Expat can be found at www.libexpat.org. # -#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI diff -Nru python3.7-3.7.0/debian/py37-breaks.Debian python3.7-3.7.1/debian/py37-breaks.Debian --- python3.7-3.7.0/debian/py37-breaks.Debian 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/debian/py37-breaks.Debian 2018-08-20 18:12:00.000000000 +0000 @@ -0,0 +1,69 @@ +python3-twisted (<< 18.4.0-2), +python3-pexpect (<< 4.6.0-1~), +python3-cxx-dev (<< 7.0.3-2), +libboost-python1.62.0 (<< 1.62.0+dfsg-6), +python3-psycopg2 (<< 2.7.5-2), +python3-maxminddb (<< 1.4.1-1~), +python3-blist (<< 1.3.6-5), +python3-django (<< 1:1.11.13-2), +diffoscope (<< 98), +gpodder (<< 3.10.3-2), +python3-trollius (<< 2.1~b1-5), +python3-slixmpp (<< 1.3.0+git20180724-1), +python3-exabgp (<< 4.0.8-1), +python3-dns (<< 3.2.0-1), +openscap-daemon (<< 0.1.10-3), +patroni (<< 1.4.4-2), +python3-cs (<< 2.3.1-1), +snakemake (<< 5.2.1-1), +python3-pycsw (<< 2.2.0+dfsg-4), +python3-aiomeasures (<< 0.5.14-2), +python3-motor (<= 1.2.3-1.1), +python3-http-parser (<= 0.8.3-2+b3), +python3-celery (<= 4.2.0-1), +python3-astroid (<= 1.6.5-1), +python3-ubjson (<= 0.8.5-2), +python3-protobuf (<= 3.0.0-9.1+b1), +python3-opengl (<= 3.1.0+dfsg-1), +python3-ws4py (<= 0.4.2+dfsg1-2), +python3-dropbox (<= 8.7.1-1), +python3-pycuda (<= 2017.1.1-2+b1), +python3-libcloud (<= 2.3.0-1), +python3-jira (<= 1.0.10-1), +python3-sleekxmpp (<= 1.3.3-3), +voltron (<= 0.1.4-2), +visidata (<= 1.0-1), +salt-common (<= 2017.7.4+dfsg1-1), +python3-yowsup (<= 2.5.7-3), +python3-websockets (<= 4.0.1-1), +python3-txfixtures (<= 0.2.6-1), +python3-tweepy (<= 3.5.0-1), +python3-signaller (<= 1.1.0-1), +python3-rpyc (<= 3.4.4-1), +python3-protorpc-standalone (<= 0.9.1-3), +python3-nova (<= 2:17.0.3-12), +python3-murano (<= 1:5.0.0-1), +python3-mastodon (<= 1.2.2-1), +python3-hug (<= 2.3.0-1.1), +python3-async (<= 0.6.2-2), +python3-pylama (<= 7.4.3-1), +python3-glance (<= 2:16.0.1-6), +python3-gbulb (<= 0.5.3-2), +python3-engineio (<= 1.6.1-1), +python3-doublex (<= 1.8.2-1), +python3-opcua (<= 0.95.1-1), +oca-core (<= 11.0.20180420-1), +python3-panoramisk (<= 1.0-1), +linux-show-player (<= 0.5-1), +python3-azure (<= 20170915+git-1), +python3-applicationinsights (<= 0.11.0-1), +python3-aiopg (<= 0.14.0-1), +python3-aiozmq (<= 0.7.1-2), +python3-pysnmp4 (<= 4.4.3-1), +python3-astroquery (<= 0.3.8+dfsg-2), +fail2ban (<= 0.10.2-2), +python3-morse-simulator (<= 1.4-3), +python3-tango (<= 9.2.3-1), +python3-pycuda (<= 2017.1.1-2), +gnome-builder (<< 3.28.3-1), +python3-pyatspi (<= 2.26.0+dfsg-1), diff -Nru python3.7-3.7.0/debian/pymindeps.py python3.7-3.7.1/debian/pymindeps.py --- python3.7-3.7.0/debian/pymindeps.py 2018-01-10 13:04:27.000000000 +0000 +++ python3.7-3.7.1/debian/pymindeps.py 2018-07-20 15:52:36.000000000 +0000 @@ -129,6 +129,7 @@ 'sysconfig': set(('pprint','_osx_support')), 'tempfile': set(('_dummy_thread', 'shutil')), 'functools': set(('typing',)), + 'platform': set(('distutils','plistlib')), } def main(argv): diff -Nru python3.7-3.7.0/debian/rules python3.7-3.7.1/debian/rules --- python3.7-3.7.0/debian/rules 2018-06-13 08:04:01.000000000 +0000 +++ python3.7-3.7.1/debian/rules 2018-09-25 14:02:49.000000000 +0000 @@ -76,9 +76,13 @@ distribution := $(shell lsb_release -is) distrelease := $(shell lsb_release -cs) +derivative := $(shell \ + if dpkg-vendor --derives-from Ubuntu; then echo Ubuntu; \ + elif dpkg-vendor --derives-from Debian; then echo Debian; \ + else echo Unknown; fi) VER=3.7 -SVER=3.7.0~b3 +SVER=3.7.0 NVER=3.8 PVER=python$(VER) EXT_VER=$(subst .,,$(VER)) @@ -156,14 +160,9 @@ OPT_CFLAGS := $(filter-out -O%,$(DPKG_CFLAGS)) # default is -O3 DEBUG_CFLAGS := $(patsubst -O%,-Og,$(DPKG_CFLAGS)) -# on alpha, use -O2 only, use -mieee -ifeq ($(DEB_HOST_ARCH),alpha) - OPT_CFLAGS += -mieee - DEBUG_CFLAGS += -mieee - EXTRA_OPT_FLAGS += -O2 -endif -ifeq ($(DEB_HOST_ARCH),m68k) - EXTRA_OPT_FLAGS += -O2 +# on ia64, disable -O3 until gcc bug #85412 is fixed +ifeq ($(DEB_HOST_ARCH),ia64) + EXTRA_OPT_CFLAGS += -O2 endif ifneq (,$(filter $(DEB_HOST_ARCH), ppc64 ppc64el)) OPT_CFLAGS += -fexceptions @@ -212,6 +211,12 @@ -e 's,^RUNSHARED *=.*,RUNSHARED=,' \ -e '/BLDLIBRARY/s/-L\. //' +ifeq ($(derivative),Ubuntu) + arch_substvars = +else ifeq ($(derivative),Debian) + arch_substvars = '-Vpython37:Breaks=$(shell tr '\n' ' ' < debian/py37-breaks.$(derivative))' +endif + make_build_target = $(if $(with_pgo),profile-opt) buildd_static := $(CURDIR)/build-static @@ -278,6 +283,10 @@ test_dbm_dumb test_dbm_ndbm test_pydoc test_sundry \ test_signal test_ioctl test_gdb test_ensurepip test_venv +# TODO: these hang/fail with new OpenSSL +PROFILE_EXCLUDES += \ + test_asyncio test_ftplib test_httplib test_imaplib test_nntplib test_poplib test_ssl + # TODO: these fail in the profile build PROFILE_EXCLUDES += \ test_cmd_line_script test_zipimport_support @@ -505,6 +514,9 @@ TEST_EXCLUDES += test_gdb endif +# fail/hang with new OpenSSL +TEST_EXCLUDES += test_asyncio test_ftplib test_httplib test_imaplib test_nntplib test_poplib test_ssl + # TODO: re-enable once fixed, see #708652 ifneq (,$(filter $(DEB_HOST_ARCH_OS), hurd)) TEST_EXCLUDES += test_asyncore test_curses test_exceptions \ @@ -1014,7 +1026,7 @@ : # The test framework into $(p_lbase) $(dh_compat2) dh_movefiles -p$(p_lbase) --sourcedir=$(d) \ $(scriptdir)/test/{libregrtest,support} \ - $(scriptdir)/test/{regrtest,test_support,__init__}.py + $(scriptdir)/test/{regrtest,test_support,__init__,__main__}.py : # The complete testsuite into $(p_lbase) $(dh_compat2) dh_movefiles -p$(p_ltst) --sourcedir=$(d) \ @@ -1438,7 +1450,7 @@ dep=`sed -n '/^shlibs:Depends/s/[^=]*=\(.*\)/\1/p' $(d_min).substvars | awk -v RS=', ' -v ORS=', ' '/^libc6/ { print }'`; \ echo "shlibs:Pre-Depends=$$dep" >> $(d_min).substvars sed -i '/^shlibs:Depends/s/libc6[^,]*[, ]*//g' $(d_min).substvars - dh_gencontrol -a + dh_gencontrol -a -- $(arch_substvars) dh_md5sums -a dh_builddeb -a diff -Nru python3.7-3.7.0/debian/tests/control python3.7-3.7.1/debian/tests/control --- python3.7-3.7.0/debian/tests/control 2018-05-30 13:13:08.000000000 +0000 +++ python3.7-3.7.1/debian/tests/control 2018-08-02 05:52:07.000000000 +0000 @@ -1,19 +1,46 @@ Tests: testsuite -Depends: build-essential, locales, python3.7-dev, libpython3.7-testsuite, python3-gdbm, python3-distutils, python3-tk, idle-python3.7 +Depends: build-essential, + locales-all, + python3.7-dev, + libpython3.7-testsuite, + python3-gdbm, + python3-distutils, + python3-tk, + idle-python3.7, # need to turn off apport Restrictions: needs-root Tests: testsuite-dbg -Depends: build-essential, locales, python3.7-dev, python3.7-dbg, libpython3.7-testsuite, python3-gdbm-dbg, gdb, python3-distutils, python3-tk-dbg, idle-python3.7 +Depends: build-essential, + locales-all, python3.7-dev, + python3.7-dbg, + libpython3.7-testsuite, + python3-gdbm-dbg, + gdb, + python3-distutils, + python3-tk-dbg, + idle-python3.7, # need to turn off apport Restrictions: needs-root Tests: failing-tests -Depends: build-essential, locales, python3.7-dev, libpython3.7-testsuite, python3-gdbm, python3-distutils +Depends: build-essential, + locales-all, + python3.7-dev, + libpython3.7-testsuite, + python3-gdbm, + python3-distutils, # need to turn off apport Restrictions: needs-root allow-stderr Tests: failing-tests-dbg -Depends: build-essential, locales, python3.7-dev, python3.7-dbg, libpython3.7-testsuite, python3-gdbm-dbg, gdb, python3-distutils +Depends: build-essential, + locales-all, + python3.7-dev, + python3.7-dbg, + libpython3.7-testsuite, + python3-gdbm-dbg, + gdb, + python3-distutils, # need to turn off apport Restrictions: needs-root allow-stderr diff -Nru python3.7-3.7.0/debian/tests/failing-tests python3.7-3.7.1/debian/tests/failing-tests --- python3.7-3.7.0/debian/tests/failing-tests 2018-05-31 10:21:19.000000000 +0000 +++ python3.7-3.7.1/debian/tests/failing-tests 2018-08-29 04:45:17.000000000 +0000 @@ -24,33 +24,14 @@ fi ls -la $ADTTMP -# no root access needed after this point - debian_dir=$(dirname $(dirname $0)) -export LOCPATH=$(pwd)/locales -sh $debian_dir/locale-gen - -export LANG=C.UTF-8 - -export DEB_PYTHON_INSTALL_LAYOUT=deb_system +# no root access needed after this point TESTPYTHON="python3.7 -W default -bb -E -R -m test" -TESTOPTS="-j 1 -w -uall,-network,-urlfetch,-gui" -TESTEXCLUSIONS= - -# test_dbm: Fails from time to time ... -#TESTEXCLUSIONS="$TESTEXCLUSIONS test_dbm" - -# test_ensurepip: not yet installed, http://bugs.debian.org/732703 -# ... and then test_venv fails too -TESTEXCLUSIONS="$TESTEXCLUSIONS test_ensurepip test_venv " - -# test_site: Investigate why this fails ... our package has one sitedir less -TESTEXCLUSIONS="$TESTEXCLUSIONS test_site" +TESTEXCLUSIONS="" -# test_lib2to3: three tests failing, investigate -TESTEXCLUSIONS="$TESTEXCLUSIONS test_lib2to3" +. $debian_dir/tests/test-common.sh if [ "$su_user" = nobody ]; then log=/dev/null diff -Nru python3.7-3.7.0/debian/tests/failing-tests-dbg python3.7-3.7.1/debian/tests/failing-tests-dbg --- python3.7-3.7.0/debian/tests/failing-tests-dbg 2018-05-31 10:21:36.000000000 +0000 +++ python3.7-3.7.1/debian/tests/failing-tests-dbg 2018-08-29 04:45:23.000000000 +0000 @@ -24,33 +24,14 @@ fi ls -la $ADTTMP -# no root access needed after this point - debian_dir=$(dirname $(dirname $0)) -export LOCPATH=$(pwd)/locales -sh $debian_dir/locale-gen - -export LANG=C.UTF-8 - -export DEB_PYTHON_INSTALL_LAYOUT=deb_system +# no root access needed after this point TESTPYTHON="python3.7dm -W default -bb -E -R -m test" -TESTOPTS="-j 1 -w -uall,-network,-urlfetch,-gui" -TESTEXCLUSIONS= - -# test_dbm: Fails from time to time ... -#TESTEXCLUSIONS="$TESTEXCLUSIONS test_dbm" - -# test_ensurepip: not yet installed, http://bugs.debian.org/732703 -# ... and then test_venv fails too -TESTEXCLUSIONS="$TESTEXCLUSIONS test_ensurepip test_venv " - -# test_site: Investigate why this fails ... our package has one sitedir less -TESTEXCLUSIONS="$TESTEXCLUSIONS test_site" +TESTEXCLUSIONS="" -# test_lib2to3: three tests failing, investigate -TESTEXCLUSIONS="$TESTEXCLUSIONS test_lib2to3" +. $debian_dir/tests/test-common.sh if [ "$su_user" = nobody ]; then log=/dev/null diff -Nru python3.7-3.7.0/debian/tests/test-common.sh python3.7-3.7.1/debian/tests/test-common.sh --- python3.7-3.7.0/debian/tests/test-common.sh 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/debian/tests/test-common.sh 2018-09-26 12:20:34.000000000 +0000 @@ -0,0 +1,47 @@ + +if dpkg-vendor --derives-from Ubuntu; then + vendor=Ubuntu +elif dpkg-vendor --derives-from Debian; then + vendor=Debian +else + vendor=Unknown +fi + +export LOCPATH=$(pwd)/locales +sh $debian_dir/locale-gen + +export LANG=C.UTF-8 + +export DEB_PYTHON_INSTALL_LAYOUT=deb_system + +TESTOPTS="-j 1 -w -uall,-network,-urlfetch,-gui" + +# test_dbm: Fails from time to time ... +#TESTEXCLUSIONS="$TESTEXCLUSIONS test_dbm" + +# test_ensurepip: not yet installed, http://bugs.debian.org/732703 +# ... and then test_venv fails too +TESTEXCLUSIONS="$TESTEXCLUSIONS test_ensurepip test_venv " + +# test_lib2to3: see https://bugs.python.org/issue34286 +TESTEXCLUSIONS="$TESTEXCLUSIONS test_lib2to3" + +# test_tcl: see https://bugs.python.org/issue34178 +TESTEXCLUSIONS="$TESTEXCLUSIONS test_tcl" + +# FIXME: Failing with OpenSSL 1.2 ... +# ssl.SSLError: [SSL: CA_MD_TOO_WEAK] ca md too weak (_ssl.c:3401) +if [ "$vendor" = Debian ]; then + TESTEXCLUSIONS="$TESTEXCLUSIONS test_asyncio test_ftplib test_httplib test_imaplib test_nntplib test_poplib test_ssl" +fi + +# FIXME: testWithTimeoutTriggeredSend: timeout not raised by _sendfile_use_sendfile +TESTEXCLUSIONS="$TESTEXCLUSIONS test_socket" + +# FIXME: issue 34806: some distutils tests fail recently +TESTEXCLUSIONS="$TESTEXCLUSIONS test_distutils" + +# FIXME, failing on the Ubuntu autopkg testers +if [ "$vendor" = Ubuntu ]; then + TESTEXCLUSIONS="$TESTEXCLUSIONS test_code_module test_platform test_site" +fi diff -Nru python3.7-3.7.0/debian/tests/testsuite python3.7-3.7.1/debian/tests/testsuite --- python3.7-3.7.0/debian/tests/testsuite 2018-05-31 10:21:44.000000000 +0000 +++ python3.7-3.7.1/debian/tests/testsuite 2018-09-26 12:19:56.000000000 +0000 @@ -1,5 +1,7 @@ #!/bin/sh +env + set -e if [ "$(whoami)" = root ]; then @@ -30,38 +32,14 @@ chown -R $su_user:nogroup $tmphome fi -# no root access needed after this point - debian_dir=$(dirname $(dirname $0)) -export LOCPATH=$(pwd)/locales -sh $debian_dir/locale-gen - -export LANG=C.UTF-8 - -export DEB_PYTHON_INSTALL_LAYOUT=deb_system +# no root access needed after this point TESTPYTHON="python3.7 -W default -bb -E -R -m test" -TESTOPTS="-j 1 -w -uall,-network,-urlfetch,-gui" TESTEXCLUSIONS="-x" -# test_dbm: Fails from time to time ... -#TESTEXCLUSIONS="$TESTEXCLUSIONS test_dbm" - -# test_ensurepip: not yet installed, http://bugs.debian.org/732703 -# ... and then test_venv fails too -TESTEXCLUSIONS="$TESTEXCLUSIONS test_ensurepip test_venv " - -# test_gdb: not run for the optimized build -TESTEXCLUSIONS="$TESTEXCLUSIONS test_gdb" - -# test_site: Investigate why this fails ... our package has one sitedir less -TESTEXCLUSIONS="$TESTEXCLUSIONS test_site" - -TESTEXCLUSIONS="$TESTEXCLUSIONS test_socket" - -# test_lib2to3: three tests failing, investigate -TESTEXCLUSIONS="$TESTEXCLUSIONS test_lib2to3" +. $debian_dir/tests/test-common.sh if [ "$su_user" = nobody ]; then log=/dev/null diff -Nru python3.7-3.7.0/debian/tests/testsuite-dbg python3.7-3.7.1/debian/tests/testsuite-dbg --- python3.7-3.7.0/debian/tests/testsuite-dbg 2018-05-31 10:21:58.000000000 +0000 +++ python3.7-3.7.1/debian/tests/testsuite-dbg 2018-08-29 04:46:00.000000000 +0000 @@ -24,35 +24,14 @@ fi ls -la $ADTTMP -# no root access needed after this point - debian_dir=$(dirname $(dirname $0)) -export LOCPATH=$(pwd)/locales -sh $debian_dir/locale-gen - -export LANG=C.UTF-8 - -export DEB_PYTHON_INSTALL_LAYOUT=deb_system +# no root access needed after this point TESTPYTHON="python3.7dm -W default -bb -E -R -m test" -TESTOPTS="-j 1 -w -uall,-network,-urlfetch,-gui" TESTEXCLUSIONS="-x" -# test_dbm: Fails from time to time ... -#TESTEXCLUSIONS="$TESTEXCLUSIONS test_dbm" - -# test_ensurepip: not yet installed, http://bugs.debian.org/732703 -# ... and then test_venv fails too -TESTEXCLUSIONS="$TESTEXCLUSIONS test_ensurepip test_venv " - -# test_site: Investigate why this fails ... our package has one sitedir less -TESTEXCLUSIONS="$TESTEXCLUSIONS test_site" - -TESTEXCLUSIONS="$TESTEXCLUSIONS test_socket" - -# test_lib2to3: three tests failing, investigate -TESTEXCLUSIONS="$TESTEXCLUSIONS test_lib2to3" +. $debian_dir/tests/test-common.sh if [ "$su_user" = nobody ]; then log=/dev/null diff -Nru python3.7-3.7.0/Doc/c-api/contextvars.rst python3.7-3.7.1/Doc/c-api/contextvars.rst --- python3.7-3.7.0/Doc/c-api/contextvars.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/c-api/contextvars.rst 2018-10-20 06:04:19.000000000 +0000 @@ -5,6 +5,25 @@ Context Variables Objects ------------------------- +.. _contextvarsobjects_pointertype_change: +.. versionchanged:: 3.7.1 + + .. note:: + + In Python 3.7.1 the signatures of all context variables + C APIs were **changed** to use :c:type:`PyObject` pointers instead + of :c:type:`PyContext`, :c:type:`PyContextVar`, and + :c:type:`PyContextToken`, e.g.:: + + // in 3.7.0: + PyContext *PyContext_New(void); + + // in 3.7.1+: + PyObject *PyContext_New(void); + + See :issue:`34762` for more details. + + .. versionadded:: 3.7 This section details the public C API for the :mod:`contextvars` module. @@ -56,27 +75,27 @@ Context object management functions: -.. c:function:: PyContext *PyContext_New(void) +.. c:function:: PyObject *PyContext_New(void) Create a new empty context object. Returns ``NULL`` if an error has occurred. -.. c:function:: PyContext *PyContext_Copy(PyContext *ctx) +.. c:function:: PyObject *PyContext_Copy(PyObject *ctx) Create a shallow copy of the passed *ctx* context object. Returns ``NULL`` if an error has occurred. -.. c:function:: PyContext *PyContext_CopyCurrent(void) +.. c:function:: PyObject *PyContext_CopyCurrent(void) Create a shallow copy of the current thread context. Returns ``NULL`` if an error has occurred. -.. c:function:: int PyContext_Enter(PyContext *ctx) +.. c:function:: int PyContext_Enter(PyObject *ctx) Set *ctx* as the current context for the current thread. Returns ``0`` on success, and ``-1`` on error. -.. c:function:: int PyContext_Exit(PyContext *ctx) +.. c:function:: int PyContext_Exit(PyObject *ctx) Deactivate the *ctx* context and restore the previous context as the current context for the current thread. Returns ``0`` on success, @@ -90,14 +109,14 @@ Context variable functions: -.. c:function:: PyContextVar *PyContextVar_New(const char *name, PyObject *def) +.. c:function:: PyObject *PyContextVar_New(const char *name, PyObject *def) Create a new ``ContextVar`` object. The *name* parameter is used for introspection and debug purposes. The *def* parameter may optionally specify the default value for the context variable. If an error has occurred, this function returns ``NULL``. -.. c:function:: int PyContextVar_Get(PyContextVar *var, PyObject *default_value, PyObject **value) +.. c:function:: int PyContextVar_Get(PyObject *var, PyObject *default_value, PyObject **value) Get the value of a context variable. Returns ``-1`` if an error has occurred during lookup, and ``0`` if no error occurred, whether or not @@ -112,13 +131,13 @@ If the value was found, the function will create a new reference to it. -.. c:function:: PyContextToken *PyContextVar_Set(PyContextVar *var, PyObject *value) +.. c:function:: PyObject *PyContextVar_Set(PyObject *var, PyObject *value) Set the value of *var* to *value* in the current context. Returns a - pointer to a :c:type:`PyContextToken` object, or ``NULL`` if an error + pointer to a :c:type:`PyObject` object, or ``NULL`` if an error has occurred. -.. c:function:: int PyContextVar_Reset(PyContextVar *var, PyContextToken *token) +.. c:function:: int PyContextVar_Reset(PyObject *var, PyObject *token) Reset the state of the *var* context variable to that it was in before :c:func:`PyContextVar_Set` that returned the *token* was called. diff -Nru python3.7-3.7.0/Doc/c-api/marshal.rst python3.7-3.7.1/Doc/c-api/marshal.rst --- python3.7-3.7.0/Doc/c-api/marshal.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/c-api/marshal.rst 2018-10-20 06:04:19.000000000 +0000 @@ -40,12 +40,6 @@ The following functions allow marshalled values to be read back in. -XXX What about error detection? It appears that reading past the end of the -file will always result in a negative numeric value (where that's relevant), -but it's not clear that negative values won't be handled properly when there's -no error. What's the right way to tell? Should only non-negative values be -written using these routines? - .. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) @@ -53,7 +47,8 @@ for reading. Only a 32-bit value can be read in using this function, regardless of the native size of :c:type:`long`. - On error, raise an exception and return ``-1``. + On error, sets the appropriate exception (:exc:`EOFError`) and returns + ``-1``. .. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) @@ -62,7 +57,8 @@ for reading. Only a 16-bit value can be read in using this function, regardless of the native size of :c:type:`short`. - On error, raise an exception and return ``-1``. + On error, sets the appropriate exception (:exc:`EOFError`) and returns + ``-1``. .. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) @@ -70,8 +66,8 @@ Return a Python object from the data stream in a :c:type:`FILE\*` opened for reading. - On error, sets the appropriate exception (:exc:`EOFError` or - :exc:`TypeError`) and returns *NULL*. + On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` + or :exc:`TypeError`) and returns *NULL*. .. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) @@ -84,8 +80,8 @@ file. Only use these variant if you are certain that you won't be reading anything else from the file. - On error, sets the appropriate exception (:exc:`EOFError` or - :exc:`TypeError`) and returns *NULL*. + On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` + or :exc:`TypeError`) and returns *NULL*. .. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) @@ -93,6 +89,6 @@ Return a Python object from the data stream in a byte buffer containing *len* bytes pointed to by *data*. - On error, sets the appropriate exception (:exc:`EOFError` or - :exc:`TypeError`) and returns *NULL*. + On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` + or :exc:`TypeError`) and returns *NULL*. diff -Nru python3.7-3.7.0/Doc/c-api/memory.rst python3.7-3.7.1/Doc/c-api/memory.rst --- python3.7-3.7.0/Doc/c-api/memory.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/c-api/memory.rst 2018-10-20 06:04:19.000000000 +0000 @@ -35,7 +35,7 @@ It is important to understand that the management of the Python heap is performed by the interpreter itself and that the user has no control over it, -even if she regularly manipulates object pointers to memory blocks inside that +even if they regularly manipulate object pointers to memory blocks inside that heap. The allocation of heap space for Python objects and other internal buffers is performed on demand by the Python memory manager through the Python/C API functions listed in this document. diff -Nru python3.7-3.7.0/Doc/conf.py python3.7-3.7.1/Doc/conf.py --- python3.7-3.7.0/Doc/conf.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/conf.py 2018-10-20 06:04:19.000000000 +0000 @@ -14,7 +14,7 @@ # --------------------- extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', - 'pyspecific', 'c_annotations'] + 'pyspecific', 'c_annotations', 'escape4chm'] # General substitutions. project = 'Python' diff -Nru python3.7-3.7.0/Doc/distutils/builtdist.rst python3.7-3.7.1/Doc/distutils/builtdist.rst --- python3.7-3.7.0/Doc/distutils/builtdist.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/distutils/builtdist.rst 2018-10-20 06:04:19.000000000 +0000 @@ -21,7 +21,7 @@ intermediary species called *packagers* springs up to turn source distributions into built distributions for as many platforms as there are packagers. -Of course, the module developer could be his own packager; or the packager could +Of course, the module developer could be their own packager; or the packager could be a volunteer "out there" somewhere who has access to a platform which the original developer does not; or it could be software periodically grabbing new source distributions and turning them into built distributions for as many diff -Nru python3.7-3.7.0/Doc/distutils/introduction.rst python3.7-3.7.1/Doc/distutils/introduction.rst --- python3.7-3.7.0/Doc/distutils/introduction.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/distutils/introduction.rst 2018-10-20 06:04:19.000000000 +0000 @@ -94,7 +94,7 @@ The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and will unpack into a directory :file:`foo-1.0`. -If an end-user wishes to install your :mod:`foo` module, all she has to do is +If an end-user wishes to install your :mod:`foo` module, all they have to do is download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and---from the :file:`foo-1.0` directory---run :: diff -Nru python3.7-3.7.0/Doc/faq/design.rst python3.7-3.7.1/Doc/faq/design.rst --- python3.7-3.7.0/Doc/faq/design.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/faq/design.rst 2018-10-20 06:04:19.000000000 +0000 @@ -2,6 +2,11 @@ Design and History FAQ ====================== +.. only:: html + + .. contents:: + + Why does Python use indentation for grouping of statements? ----------------------------------------------------------- @@ -210,24 +215,25 @@ Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? ---------------------------------------------------------------------------------------------------------------- -The major reason is history. Functions were used for those operations that were -generic for a group of types and which were intended to work even for objects -that didn't have methods at all (e.g. tuples). It is also convenient to have a -function that can readily be applied to an amorphous collection of objects when -you use the functional features of Python (``map()``, ``zip()`` et al). - -In fact, implementing ``len()``, ``max()``, ``min()`` as a built-in function is -actually less code than implementing them as methods for each type. One can -quibble about individual cases but it's a part of Python, and it's too late to -make such fundamental changes now. The functions have to remain to avoid massive -code breakage. +As Guido said: -.. XXX talk about protocols? + (a) For some operations, prefix notation just reads better than + postfix -- prefix (and infix!) operations have a long tradition in + mathematics which likes notations where the visuals help the + mathematician thinking about a problem. Compare the easy with which we + rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of + doing the same thing using a raw OO notation. + + (b) When I read code that says len(x) I *know* that it is asking for + the length of something. This tells me two things: the result is an + integer, and the argument is some kind of container. To the contrary, + when I read x.len(), I have to already know that x is some kind of + container implementing an interface or inheriting from a class that + has a standard len(). Witness the confusion we occasionally have when + a class that is not implementing a mapping has a get() or keys() + method, or something that isn't a file has a write() method. -.. note:: - - For string operations, Python has moved from external functions (the - ``string`` module) to methods. However, ``len()`` is still a function. + -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html Why is join() a string method instead of a list or tuple method? @@ -343,7 +349,7 @@ random moments. Therefore, a complete threads implementation requires thread support for C. -Answer 2: Fortunately, there is `Stackless Python `_, +Answer 2: Fortunately, there is `Stackless Python `_, which has a completely redesigned interpreter loop that avoids the C stack. @@ -465,10 +471,10 @@ dictionary keys, and hence only tuples and not lists can be used as keys. -How are lists implemented? --------------------------- +How are lists implemented in CPython? +------------------------------------- -Python's lists are really variable-length arrays, not Lisp-style linked lists. +CPython's lists are really variable-length arrays, not Lisp-style linked lists. The implementation uses a contiguous array of references to other objects, and keeps a pointer to this array and the array's length in a list head structure. @@ -481,10 +487,10 @@ times don't require an actual resize. -How are dictionaries implemented? ---------------------------------- +How are dictionaries implemented in CPython? +-------------------------------------------- -Python's dictionaries are implemented as resizable hash tables. Compared to +CPython's dictionaries are implemented as resizable hash tables. Compared to B-trees, this gives better performance for lookup (the most common operation by far) under most circumstances, and the implementation is simpler. @@ -495,7 +501,7 @@ to 1142331976. The hash code is then used to calculate a location in an internal array where the value will be stored. Assuming that you're storing keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in computer science notation -- to retrieve a key. +constant time -- O(1), in Big-O notation -- to retrieve a key. Why must dictionary keys be immutable? diff -Nru python3.7-3.7.0/Doc/faq/general.rst python3.7-3.7.1/Doc/faq/general.rst --- python3.7-3.7.0/Doc/faq/general.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/faq/general.rst 2018-10-20 06:04:19.000000000 +0000 @@ -306,17 +306,19 @@ The developers issue "bugfix" releases of older versions, so the stability of existing releases gradually improves. Bugfix releases, indicated by a third -component of the version number (e.g. 2.5.3, 2.6.2), are managed for stability; +component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability; only fixes for known problems are included in a bugfix release, and it's guaranteed that interfaces will remain the same throughout a series of bugfix releases. The latest stable releases can always be found on the `Python download page -`_. There are two recommended production-ready -versions at this point in time, because at the moment there are two branches of -stable releases: 2.x and 3.x. Python 3.x may be less useful than 2.x, since -currently there is more third party software available for Python 2 than for -Python 3. Python 2 code will generally not run unchanged in Python 3. +`_. There are two production-ready version +of Python: 2.x and 3.x, but the recommended one at this times is Python 3.x. +Although Python 2.x is still widely used, `it will not be +maintained after January 1, 2020 `_. +Python 2.x was known for having more third-party libraries available, however, +by the time of this writing, most of the widely used libraries support Python 3.x, +and some are even dropping the Python 2.x support. How many people are using Python? diff -Nru python3.7-3.7.0/Doc/faq/programming.rst python3.7-3.7.1/Doc/faq/programming.rst --- python3.7-3.7.0/Doc/faq/programming.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/faq/programming.rst 2018-10-20 06:04:19.000000000 +0000 @@ -71,6 +71,11 @@ standard, whether declared interfaces are fully implemented, and more. https://docs.pylint.org/ provides a full list of Pylint's features. +Static type checkers such as `Mypy `_, +`Pyre `_, and +`Pytype `_ can check type hints in Python +source code. + How can I create a stand-alone binary from a Python script? ----------------------------------------------------------- @@ -371,8 +376,8 @@ of each call to the function, and return the cached value if the same value is requested again. This is called "memoizing", and can be implemented like this:: - # Callers will never provide a third parameter for this function. - def expensive(arg1, arg2, _cache={}): + # Callers can only provide two parameters and optionally pass _cache by keyword + def expensive(arg1, arg2, *, _cache={}): if (arg1, arg2) in _cache: return _cache[(arg1, arg2)] diff -Nru python3.7-3.7.0/Doc/glossary.rst python3.7-3.7.1/Doc/glossary.rst --- python3.7-3.7.0/Doc/glossary.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/glossary.rst 2018-10-20 06:04:19.000000000 +0000 @@ -14,8 +14,9 @@ ``...`` The default Python prompt of the interactive shell when entering code for - an indented code block or within a pair of matching left and right - delimiters (parentheses, square brackets or curly braces). + an indented code block, when within a pair of matching left and right + delimiters (parentheses, square brackets, curly braces or triple quotes), + or after specifying a decorator. 2to3 A tool that tries to convert Python 2.x code to Python 3.x code by @@ -122,10 +123,10 @@ :meth:`__aiter__` method. Introduced by :pep:`492`. asynchronous iterator - An object that implements :meth:`__aiter__` and :meth:`__anext__` + An object that implements the :meth:`__aiter__` and :meth:`__anext__` methods. ``__anext__`` must return an :term:`awaitable` object. - :keyword:`async for` resolves awaitable returned from asynchronous - iterator's :meth:`__anext__` method until it raises + :keyword:`async for` resolves the awaitables returned by an asynchronous + iterator's :meth:`__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. attribute @@ -642,7 +643,7 @@ list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to - elements are O(1). + elements is O(1). list comprehension A compact way to process all or part of the elements in a sequence and @@ -1011,7 +1012,7 @@ struct sequence A tuple with named elements. Struct sequences expose an interface similar - to :term:`named tuple` in that elements can either be accessed either by + to :term:`named tuple` in that elements can be accessed either by index or as an attribute. However, they do not have any of the named tuple methods like :meth:`~collections.somenamedtuple._make` or :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences diff -Nru python3.7-3.7.0/Doc/howto/descriptor.rst python3.7-3.7.1/Doc/howto/descriptor.rst --- python3.7-3.7.0/Doc/howto/descriptor.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/howto/descriptor.rst 2018-10-20 06:04:19.000000000 +0000 @@ -11,7 +11,7 @@ -------- Defines descriptors, summarizes the protocol, and shows how descriptors are -called. Examines a custom descriptor and several built-in python descriptors +called. Examines a custom descriptor and several built-in Python descriptors including functions, properties, static methods, and class methods. Shows how each works by giving a pure Python equivalent and a sample application. @@ -48,11 +48,11 @@ Descriptor Protocol ------------------- -``descr.__get__(self, obj, type=None) --> value`` +``descr.__get__(self, obj, type=None) -> value`` -``descr.__set__(self, obj, value) --> None`` +``descr.__set__(self, obj, value) -> None`` -``descr.__delete__(self, obj) --> None`` +``descr.__delete__(self, obj) -> None`` That is all there is to it. Define any of these methods and an object is considered a descriptor and can override default behavior upon being looked up @@ -275,7 +275,7 @@ To support method calls, functions include the :meth:`__get__` method for binding methods during attribute access. This means that all functions are non-data descriptors which return bound methods when they are invoked from an -object. In pure python, it works like this:: +object. In pure Python, it works like this:: class Function(object): . . . diff -Nru python3.7-3.7.0/Doc/howto/functional.rst python3.7-3.7.1/Doc/howto/functional.rst --- python3.7-3.7.0/Doc/howto/functional.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/howto/functional.rst 2018-10-20 06:04:19.000000000 +0000 @@ -198,7 +198,7 @@ You can experiment with the iteration interface manually: - >>> L = [1,2,3] + >>> L = [1, 2, 3] >>> it = iter(L) >>> it #doctest: +ELLIPSIS <...iterator object at ...> @@ -229,7 +229,7 @@ Iterators can be materialized as lists or tuples by using the :func:`list` or :func:`tuple` constructor functions: - >>> L = [1,2,3] + >>> L = [1, 2, 3] >>> iterator = iter(L) >>> t = tuple(iterator) >>> t @@ -238,10 +238,10 @@ Sequence unpacking also supports iterators: if you know an iterator will return N elements, you can unpack them into an N-tuple: - >>> L = [1,2,3] + >>> L = [1, 2, 3] >>> iterator = iter(L) - >>> a,b,c = iterator - >>> a,b,c + >>> a, b, c = iterator + >>> a, b, c (1, 2, 3) Built-in functions such as :func:`max` and :func:`min` can take a single @@ -273,23 +273,24 @@ >>> m = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, ... 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} - >>> for key in m: #doctest: +SKIP + >>> for key in m: ... print(key, m[key]) - Mar 3 + Jan 1 Feb 2 - Aug 8 - Sep 9 + Mar 3 Apr 4 + May 5 Jun 6 Jul 7 - Jan 1 - May 5 + Aug 8 + Sep 9 + Oct 10 Nov 11 Dec 12 - Oct 10 -Note that the order is essentially random, because it's based on the hash -ordering of the objects in the dictionary. +Note that starting with Python 3.7, dictionary iteration order is guaranteed +to be the same as the insertion order. In earlier versions, the behaviour was +unspecified and could vary between implementations. Applying :func:`iter` to a dictionary always loops over the keys, but dictionaries have methods that return other iterators. If you want to iterate @@ -301,8 +302,8 @@ of ``(key, value)`` tuples: >>> L = [('Italy', 'Rome'), ('France', 'Paris'), ('US', 'Washington DC')] - >>> dict(iter(L)) #doctest: +SKIP - {'Italy': 'Rome', 'US': 'Washington DC', 'France': 'Paris'} + >>> dict(iter(L)) + {'Italy': 'Rome', 'France': 'Paris', 'US': 'Washington DC'} Files also support iteration by calling the :meth:`~io.TextIOBase.readline` method until there are no more lines in the file. This means you can read each @@ -410,7 +411,7 @@ list is 9 elements long: >>> seq1 = 'abc' - >>> seq2 = (1,2,3) + >>> seq2 = (1, 2, 3) >>> [(x, y) for x in seq1 for y in seq2] #doctest: +NORMALIZE_WHITESPACE [('a', 1), ('a', 2), ('a', 3), ('b', 1), ('b', 2), ('b', 3), @@ -478,7 +479,7 @@ File "stdin", line 2, in generate_ints StopIteration -You could equally write ``for i in generate_ints(5)``, or ``a,b,c = +You could equally write ``for i in generate_ints(5)``, or ``a, b, c = generate_ints(3)``. Inside a generator function, ``return value`` causes ``StopIteration(value)`` @@ -694,17 +695,17 @@ in the iterable is a true value, and :func:`all` returns ``True`` if all of the elements are true values: - >>> any([0,1,0]) + >>> any([0, 1, 0]) True - >>> any([0,0,0]) + >>> any([0, 0, 0]) False - >>> any([1,1,1]) + >>> any([1, 1, 1]) True - >>> all([0,1,0]) + >>> all([0, 1, 0]) False - >>> all([0,0,0]) + >>> all([0, 0, 0]) False - >>> all([1,1,1]) + >>> all([1, 1, 1]) True @@ -763,7 +764,7 @@ a provided iterable and returns a new iterator that returns its elements from first to last. The new iterator will repeat these elements infinitely. :: - itertools.cycle([1,2,3,4,5]) => + itertools.cycle([1, 2, 3, 4, 5]) => 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, ... :func:`itertools.repeat(elem, [n]) ` returns the provided @@ -874,7 +875,7 @@ iterators and returns only those elements of *data* for which the corresponding element of *selectors* is true, stopping whenever either one is exhausted:: - itertools.compress([1,2,3,4,5], [True, True, False, False, True]) => + itertools.compress([1, 2, 3, 4, 5], [True, True, False, False, True]) => 1, 2, 5 @@ -1034,7 +1035,7 @@ Traceback (most recent call last): ... TypeError: reduce() of empty sequence with no initial value - >>> functools.reduce(operator.mul, [1,2,3], 1) + >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) 1 @@ -1044,9 +1045,9 @@ built-in called :func:`sum` to compute it: >>> import functools, operator - >>> functools.reduce(operator.add, [1,2,3,4], 0) + >>> functools.reduce(operator.add, [1, 2, 3, 4], 0) 10 - >>> sum([1,2,3,4]) + >>> sum([1, 2, 3, 4]) 10 >>> sum([]) 0 @@ -1056,11 +1057,11 @@ import functools # Instead of: - product = functools.reduce(operator.mul, [1,2,3], 1) + product = functools.reduce(operator.mul, [1, 2, 3], 1) # You can write: product = 1 - for i in [1,2,3]: + for i in [1, 2, 3]: product *= i A related function is :func:`itertools.accumulate(iterable, func=operator.add) @@ -1068,10 +1069,10 @@ returning only the final result, :func:`accumulate` returns an iterator that also yields each partial result:: - itertools.accumulate([1,2,3,4,5]) => + itertools.accumulate([1, 2, 3, 4, 5]) => 1, 3, 6, 10, 15 - itertools.accumulate([1,2,3,4,5], operator.mul) => + itertools.accumulate([1, 2, 3, 4, 5], operator.mul) => 1, 2, 6, 24, 120 @@ -1155,7 +1156,7 @@ Or the :func:`sum` built-in and a generator expression:: - total = sum(b for a,b in items) + total = sum(b for a, b in items) Many uses of :func:`functools.reduce` are clearer when written as ``for`` loops. diff -Nru python3.7-3.7.0/Doc/howto/instrumentation.rst python3.7-3.7.1/Doc/howto/instrumentation.rst --- python3.7-3.7.0/Doc/howto/instrumentation.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/howto/instrumentation.rst 2018-10-20 06:04:19.000000000 +0000 @@ -369,13 +369,13 @@ .. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr) This probe point indicates that execution of a Python function has begun. - It is only triggered for pure-python (bytecode) functions. + It is only triggered for pure-Python (bytecode) functions. .. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr) This probe point is the converse of :c:func:`python.function.return`, and indicates that execution of a Python function has ended (either via - ``return``, or via an exception). It is only triggered for pure-python + ``return``, or via an exception). It is only triggered for pure-Python (bytecode) functions. diff -Nru python3.7-3.7.0/Doc/includes/run-func.c python3.7-3.7.1/Doc/includes/run-func.c --- python3.7-3.7.0/Doc/includes/run-func.c 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/includes/run-func.c 2018-10-20 06:04:19.000000000 +0000 @@ -3,7 +3,7 @@ int main(int argc, char *argv[]) { - PyObject *pName, *pModule, *pDict, *pFunc; + PyObject *pName, *pModule, *pFunc; PyObject *pArgs, *pValue; int i; diff -Nru python3.7-3.7.0/Doc/includes/tzinfo_examples.py python3.7-3.7.1/Doc/includes/tzinfo_examples.py --- python3.7-3.7.0/Doc/includes/tzinfo_examples.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/includes/tzinfo_examples.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,4 +1,4 @@ -from datetime import tzinfo, timedelta, datetime, timezone +from datetime import tzinfo, timedelta, datetime ZERO = timedelta(0) HOUR = timedelta(hours=1) diff -Nru python3.7-3.7.0/Doc/library/2to3.rst python3.7-3.7.1/Doc/library/2to3.rst --- python3.7-3.7.0/Doc/library/2to3.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/2to3.rst 2018-10-20 06:04:19.000000000 +0000 @@ -385,7 +385,7 @@ .. 2to3fixer:: reload - Converts :func:`reload` to :func:`imp.reload`. + Converts :func:`reload` to :func:`importlib.reload`. .. 2to3fixer:: renames diff -Nru python3.7-3.7.0/Doc/library/argparse.rst python3.7-3.7.1/Doc/library/argparse.rst --- python3.7-3.7.0/Doc/library/argparse.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/argparse.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1539,7 +1539,7 @@ .. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ [parser_class], [action], \ - [option_string], [dest], [required] \ + [option_string], [dest], [required], \ [help], [metavar]) Many programs split up their functionality into a number of sub-commands, diff -Nru python3.7-3.7.0/Doc/library/asyncio-api-index.rst python3.7-3.7.1/Doc/library/asyncio-api-index.rst --- python3.7-3.7.0/Doc/library/asyncio-api-index.rst 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-api-index.rst 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,218 @@ +.. currentmodule:: asyncio + + +==================== +High-level API Index +==================== + +This page lists all high-level async/await enabled asyncio APIs. + + +Tasks +===== + +Utilities to run asyncio programs, create Tasks, and +await on multiple things with timeouts. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :func:`run` + - Create event loop, run a coroutine, close the loop. + + * - :func:`create_task` + - Start an asyncio Task. + + * - ``await`` :func:`sleep` + - Sleep for a number of seconds. + + * - ``await`` :func:`gather` + - Schedule and wait for things concurrently. + + * - ``await`` :func:`wait_for` + - Run with a timeout. + + * - ``await`` :func:`shield` + - Shield from cancellation. + + * - ``await`` :func:`wait` + - Monitor for completion. + + * - :func:`current_task` + - Return the current Task. + + * - :func:`all_tasks` + - Return all tasks for an event loop. + + * - :class:`Task` + - Task object. + + * - :func:`run_coroutine_threadsafe` + - Schedule a coroutine from another OS thread. + + * - ``for in`` :func:`as_completed` + - Monitor for completion with a ``for`` loop. + + +.. rubric:: Examples + +* :ref:`Using asyncio.gather() to run things in parallel + `. + +* :ref:`Using asyncio.wait_for() to enforce a timeout + `. + +* :ref:`Cancellation `. + +* :ref:`Using asyncio.sleep() `. + +* See also the main :ref:`Tasks documentation page `. + + +Queues +====== + +Queues should be used to distribute work amongst multiple asyncio Tasks, +implement connection pools, and pub/sub patterns. + + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Queue` + - A FIFO queue. + + * - :class:`PriorityQueue` + - A priority queue. + + * - :class:`LifoQueue` + - A LIFO queue. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Queue to distribute workload between several + Tasks `. + +* See also the :ref:`Queues documentation page `. + + +Subprocesses +============ + +Utilities to spawn subprocesses and run shell commands. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`create_subprocess_exec` + - Create a subprocess. + + * - ``await`` :func:`create_subprocess_shell` + - Run a shell command. + + +.. rubric:: Examples + +* :ref:`Executing a shell command `. + +* See also the :ref:`subprocess APIs ` + documentation. + + +Streams +======= + +High-level APIs to work with network IO. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`open_connection` + - Establish a TCP connection. + + * - ``await`` :func:`open_unix_connection` + - Establish a Unix socket connection. + + * - ``await`` :func:`start_server` + - Start a TCP server. + + * - ``await`` :func:`start_unix_server` + - Start a Unix socket server. + + * - :class:`StreamReader` + - High-level async/await object to receive network data. + + * - :class:`StreamWriter` + - High-level async/await object to send network data. + + +.. rubric:: Examples + +* :ref:`Example TCP client `. + +* See also the :ref:`streams APIs ` + documentation. + + +Synchronization +=============== + +Threading-like synchronization primitives that can be used in Tasks. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Lock` + - A mutex lock. + + * - :class:`Event` + - An event object. + + * - :class:`Condition` + - A condition object. + + * - :class:`Semaphore` + - A semaphore. + + * - :class:`BoundedSemaphore` + - A bounded semaphore. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Event `. + +* See also the documentation of asyncio + :ref:`synchronization primitives `. + + +Exceptions +========== + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + + * - :exc:`asyncio.TimeoutError` + - Raised on timeout by functions like :func:`wait_for`. + Keep in mind that ``asyncio.TimeoutError`` is **unrelated** + to the built-in :exc:`TimeoutError` exception. + + * - :exc:`asyncio.CancelledError` + - Raised when a Task is cancelled. See also :meth:`Task.cancel`. + + +.. rubric:: Examples + +* :ref:`Handling CancelledError to run code on cancellation request + `. + +* See also the full list of + :ref:`asyncio-specific exceptions `. diff -Nru python3.7-3.7.0/Doc/library/asyncio-dev.rst python3.7-3.7.1/Doc/library/asyncio-dev.rst --- python3.7-3.7.0/Doc/library/asyncio-dev.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-dev.rst 2018-10-20 06:04:19.000000000 +0000 @@ -2,415 +2,236 @@ .. _asyncio-dev: -Develop with asyncio -==================== +======================= +Developing with asyncio +======================= -Asynchronous programming is different than classical "sequential" programming. -This page lists common traps and explains how to avoid them. +Asynchronous programming is different from classic "sequential" +programming. + +This page lists common mistakes and traps and explains how +to avoid them. .. _asyncio-debug-mode: -Debug mode of asyncio ---------------------- +Debug Mode +========== -The implementation of :mod:`asyncio` has been written for performance. -In order to ease the development of asynchronous code, you may wish to -enable *debug mode*. - -To enable all debug checks for an application: - -* Enable the asyncio debug mode globally by setting the environment variable - :envvar:`PYTHONASYNCIODEBUG` to ``1``, using ``-X dev`` command line option - (see the :option:`-X` option), or by calling - :meth:`AbstractEventLoop.set_debug`. -* Set the log level of the :ref:`asyncio logger ` to - :py:data:`logging.DEBUG`. For example, call - ``logging.basicConfig(level=logging.DEBUG)`` at startup. -* Configure the :mod:`warnings` module to display :exc:`ResourceWarning` - warnings. For example, use the ``-Wdefault`` command line option of Python to - display them. - -Examples debug checks: - -* Log :ref:`coroutines defined but never "yielded from" - ` -* :meth:`~AbstractEventLoop.call_soon` and :meth:`~AbstractEventLoop.call_at` methods - raise an exception if they are called from the wrong thread. -* Log the execution time of the selector -* Log callbacks taking more than 100 ms to be executed. The - :attr:`AbstractEventLoop.slow_callback_duration` attribute is the minimum - duration in seconds of "slow" callbacks. -* :exc:`ResourceWarning` warnings are emitted when transports and event loops - are :ref:`not closed explicitly `. - -.. versionchanged:: 3.7 - - The new ``-X dev`` command line option can now also be used to enable - the debug mode. - -.. seealso:: - - The :meth:`AbstractEventLoop.set_debug` method and the :ref:`asyncio logger - `. - - -Cancellation ------------- - -Cancellation of tasks is not common in classic programming. In asynchronous -programming, not only is it something common, but you have to prepare your -code to handle it. - -Futures and tasks can be cancelled explicitly with their :meth:`Future.cancel` -method. The :func:`wait_for` function cancels the waited task when the timeout -occurs. There are many other cases where a task can be cancelled indirectly. - -Don't call :meth:`~Future.set_result` or :meth:`~Future.set_exception` method -of :class:`Future` if the future is cancelled: it would fail with an exception. -For example, write:: - - if not fut.cancelled(): - fut.set_result('done') - -Don't schedule directly a call to the :meth:`~Future.set_result` or the -:meth:`~Future.set_exception` method of a future with -:meth:`AbstractEventLoop.call_soon`: the future can be cancelled before its method -is called. - -If you wait for a future, you should check early if the future was cancelled to -avoid useless operations. Example:: - - async def slow_operation(fut): - if fut.cancelled(): - return - # ... slow computation ... - await fut - # ... +By default asyncio runs in production mode. In order to ease +the development asyncio has a *debug mode*. -The :func:`shield` function can also be used to ignore cancellation. +There are several ways to enable asyncio debug mode: +* Setting the :envvar:`PYTHONASYNCIODEBUG` environment variable to ``1``. -.. _asyncio-multithreading: +* Using the :option:`-X` ``dev`` Python command line option. -Concurrency and multithreading ------------------------------- +* Passing ``debug=True`` to :func:`asyncio.run`. -An event loop runs in a thread and executes all callbacks and tasks in the same -thread. While a task is running in the event loop, no other task is running in -the same thread. But when the task uses ``await``, the task is suspended -and the event loop executes the next task. +* Calling :meth:`loop.set_debug`. -To schedule a callback from a different thread, the -:meth:`AbstractEventLoop.call_soon_threadsafe` method should be used. Example:: +In addition to enabling the debug mode, consider also: - loop.call_soon_threadsafe(callback, *args) +* setting the log level of the :ref:`asyncio logger ` to + :py:data:`logging.DEBUG`, for example the following snippet of code + can be run at startup of the application:: -Most asyncio objects are not thread safe. You should only worry if you access -objects outside the event loop. For example, to cancel a future, don't call -directly its :meth:`Future.cancel` method, but:: + logging.basicConfig(level=logging.DEBUG) - loop.call_soon_threadsafe(fut.cancel) +* configuring the :mod:`warnings` module to display + :exc:`ResourceWarning` warnings. One way of doing that is by + using the :option:`-W` ``default`` command line option. -To handle signals and to execute subprocesses, the event loop must be run in -the main thread. -To schedule a coroutine object from a different thread, the -:func:`run_coroutine_threadsafe` function should be used. It returns a -:class:`concurrent.futures.Future` to access the result:: +When the debug mode is enabled: - future = asyncio.run_coroutine_threadsafe(coro_func(), loop) - result = future.result(timeout) # Wait for the result with a timeout +* asyncio checks for :ref:`coroutines that were not awaited + ` and logs them; this mitigates + the "forgotten await" pitfall. -The :meth:`AbstractEventLoop.run_in_executor` method can be used with a thread pool -executor to execute a callback in different thread to not block the thread of -the event loop. +* Many non-treadsafe asyncio APIs (such as :meth:`loop.call_soon` and + :meth:`loop.call_at` methods) raise an exception if they are called + from a wrong thread. -.. seealso:: +* The execution time of the I/O selector is logged if it takes too long to + perform an I/O operation. - The :ref:`Synchronization primitives ` section describes ways - to synchronize tasks. +* Callbacks taking longer than 100ms are logged. The + :attr:`loop.slow_callback_duration` attribute can be used to set the + minimum execution duration in seconds that is considered "slow". - The :ref:`Subprocess and threads ` section lists - asyncio limitations to run subprocesses from different threads. +.. _asyncio-multithreading: +Concurrency and Multithreading +============================== +An event loop runs in a thread (typically the main thread) and executes +all callbacks and Tasks in its thread. While a Task is running in the +event loop, no other Tasks can run in the same thread. When a Task +executes an ``await`` expression, the running Task gets suspended, and +the event loop executes the next Task. -.. _asyncio-handle-blocking: +To schedule a callback from a different OS thread, the +:meth:`loop.call_soon_threadsafe` method should be used. Example:: + + loop.call_soon_threadsafe(callback, *args) + +Almost all asyncio objects are not thread safe, which is typically +not a problem unless there is code that works with them from outside +of a Task or a callback. If there's a need for such code to call a +low-level asyncio API, the :meth:`loop.call_soon_threadsafe` method +should be used, e.g.:: + + loop.call_soon_threadsafe(fut.cancel) + +To schedule a coroutine object from a different OS thread, the +:func:`run_coroutine_threadsafe` function should be used. It returns a +:class:`concurrent.futures.Future` to access the result:: -Handle blocking functions correctly ------------------------------------ + async def coro_func(): + return await asyncio.sleep(1, 42) -Blocking functions should not be called directly. For example, if a function -blocks for 1 second, other tasks are delayed by 1 second which can have an -important impact on reactivity. + # Later in another OS thread: -For networking and subprocesses, the :mod:`asyncio` module provides high-level -APIs like :ref:`protocols `. + future = asyncio.run_coroutine_threadsafe(coro_func(), loop) + # Wait for the result: + result = future.result() -An executor can be used to run a task in a different thread or even in a -different process, to not block the thread of the event loop. See the -:meth:`AbstractEventLoop.run_in_executor` method. +To handle signals and to execute subprocesses, the event loop must be +run in the main thread. -.. seealso:: +The :meth:`loop.run_in_executor` method can be used with a +:class:`concurrent.futures.ThreadPoolExecutor` to execute +blocking code in a different OS thread without blocking the OS thread +that the event loop runs in. - The :ref:`Delayed calls ` section details how the - event loop handles time. + +.. _asyncio-handle-blocking: + +Running Blocking Code +===================== + +Blocking (CPU-bound) code should not be called directly. For example, +if a function performs a CPU-intensive calculation for 1 second, +all concurrent asyncio Tasks and IO operations would be delayed +by 1 second. + +An executor can be used to run a task in a different thread or even in +a different process to avoid blocking block the OS thread with the +event loop. See the :meth:`loop.run_in_executor` method for more +details. .. _asyncio-logger: Logging -------- - -The :mod:`asyncio` module logs information with the :mod:`logging` module in -the logger ``'asyncio'``. +======= -The default log level for the :mod:`asyncio` module is :py:data:`logging.INFO`. -For those not wanting such verbosity from :mod:`asyncio` the log level can -be changed. For example, to change the level to :py:data:`logging.WARNING`: +asyncio uses the :mod:`logging` module and all logging is performed +via the ``"asyncio"`` logger. -.. code-block:: none +The default log level is :py:data:`logging.INFO`, which can be easily +adjusted:: - logging.getLogger('asyncio').setLevel(logging.WARNING) + logging.getLogger("asyncio").setLevel(logging.WARNING) .. _asyncio-coroutine-not-scheduled: -Detect coroutine objects never scheduled ----------------------------------------- +Detect never-awaited coroutines +=============================== -When a coroutine function is called and its result is not passed to -:func:`ensure_future` or to the :meth:`AbstractEventLoop.create_task` method, -the execution of the coroutine object will never be scheduled which is -probably a bug. :ref:`Enable the debug mode of asyncio ` -to :ref:`log a warning ` to detect it. - -Example with the bug:: +When a coroutine function is called, but not awaited +(e.g. ``coro()`` instead of ``await coro()``) +or the coroutine is not scheduled with :meth:`asyncio.create_task`, asyncio +will emit a :exc:`RuntimeWarning`:: import asyncio async def test(): print("never scheduled") + async def main(): + test() + + asyncio.run(main()) + +Output:: + + test.py:7: RuntimeWarning: coroutine 'test' was never awaited test() Output in debug mode:: - Coroutine test() at test.py:3 was never yielded from - Coroutine object created at (most recent call last): - File "test.py", line 7, in - test() + test.py:7: RuntimeWarning: coroutine 'test' was never awaited + Coroutine created at (most recent call last) + File "../t.py", line 9, in + asyncio.run(main(), debug=True) -The fix is to call the :func:`ensure_future` function or the -:meth:`AbstractEventLoop.create_task` method with the coroutine object. + < .. > -.. seealso:: + File "../t.py", line 7, in main + test() + test() + +The usual fix is to either await the coroutine or call the +:meth:`asyncio.create_task` function:: - :ref:`Pending task destroyed `. + async def main(): + await test() -Detect exceptions never consumed --------------------------------- +Detect never-retrieved exceptions +================================= -Python usually calls :func:`sys.excepthook` on unhandled exceptions. If -:meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted -` when the future is deleted by the garbage collector, with the -traceback where the exception was raised. +If a :meth:`Future.set_exception` is called but the Future object is +never awaited on, the exception would never be propagated to the +user code. In this case, asyncio would emit a log message when the +Future object is garbage collected. -Example of unhandled exception:: +Example of an unhandled exception:: import asyncio - @asyncio.coroutine - def bug(): + async def bug(): raise Exception("not consumed") - loop = asyncio.get_event_loop() - asyncio.ensure_future(bug()) - loop.run_forever() - loop.close() + async def main(): + asyncio.create_task(bug()) + + asyncio.run(main()) Output:: Task exception was never retrieved - future: exception=Exception('not consumed',)> - Traceback (most recent call last): - File "asyncio/tasks.py", line 237, in _step - result = next(coro) - File "asyncio/coroutines.py", line 141, in coro - res = func(*args, **kw) - File "test.py", line 5, in bug - raise Exception("not consumed") - Exception: not consumed - -:ref:`Enable the debug mode of asyncio ` to get the -traceback where the task was created. Output in debug mode:: + future: + exception=Exception('not consumed')> - Task exception was never retrieved - future: exception=Exception('not consumed',) created at test.py:8> - source_traceback: Object created at (most recent call last): - File "test.py", line 8, in - asyncio.ensure_future(bug()) Traceback (most recent call last): - File "asyncio/tasks.py", line 237, in _step - result = next(coro) - File "asyncio/coroutines.py", line 79, in __next__ - return next(self.gen) - File "asyncio/coroutines.py", line 141, in coro - res = func(*args, **kw) - File "test.py", line 5, in bug + File "test.py", line 4, in bug raise Exception("not consumed") Exception: not consumed -There are different options to fix this issue. The first option is to chain the -coroutine in another coroutine and use classic try/except:: +:ref:`Enable the debug mode ` to get the +traceback where the task was created:: - async def handle_exception(): - try: - await bug() - except Exception: - print("exception consumed") + asyncio.run(main(), debug=True) - loop = asyncio.get_event_loop() - asyncio.ensure_future(handle_exception()) - loop.run_forever() - loop.close() - -Another option is to use the :meth:`AbstractEventLoop.run_until_complete` -function:: - - task = asyncio.ensure_future(bug()) - try: - loop.run_until_complete(task) - except Exception: - print("exception consumed") - -.. seealso:: - - The :meth:`Future.exception` method. - - -Chain coroutines correctly --------------------------- - -When a coroutine function calls other coroutine functions and tasks, they -should be chained explicitly with ``await``. Otherwise, the execution is -not guaranteed to be sequential. - -Example with different bugs using :func:`asyncio.sleep` to simulate slow -operations:: - - import asyncio - - async def create(): - await asyncio.sleep(3.0) - print("(1) create file") - - async def write(): - await asyncio.sleep(1.0) - print("(2) write into file") - - async def close(): - print("(3) close file") - - async def test(): - asyncio.ensure_future(create()) - asyncio.ensure_future(write()) - asyncio.ensure_future(close()) - await asyncio.sleep(2.0) - loop.stop() - - loop = asyncio.get_event_loop() - asyncio.ensure_future(test()) - loop.run_forever() - print("Pending tasks at exit: %s" % asyncio.Task.all_tasks(loop)) - loop.close() - -Expected output: - -.. code-block:: none - - (1) create file - (2) write into file - (3) close file - Pending tasks at exit: set() - -Actual output: - -.. code-block:: none - - (3) close file - (2) write into file - Pending tasks at exit: {>} - Task was destroyed but it is pending! - task: > - -The loop stopped before the ``create()`` finished, ``close()`` has been called -before ``write()``, whereas coroutine functions were called in this order: -``create()``, ``write()``, ``close()``. - -To fix the example, tasks must be marked with ``await``:: - - async def test(): - await asyncio.ensure_future(create()) - await asyncio.ensure_future(write()) - await asyncio.ensure_future(close()) - await asyncio.sleep(2.0) - loop.stop() - -Or without ``asyncio.ensure_future()``:: - - async def test(): - await create() - await write() - await close() - await asyncio.sleep(2.0) - loop.stop() - - -.. _asyncio-pending-task-destroyed: - -Pending task destroyed ----------------------- - -If a pending task is destroyed, the execution of its wrapped :ref:`coroutine -` did not complete. It is probably a bug and so a warning is logged. - -Example of log: - -.. code-block:: none - - Task was destroyed but it is pending! - task: wait_for=> - -:ref:`Enable the debug mode of asyncio ` to get the -traceback where the task was created. Example of log in debug mode: +Output in debug mode:: -.. code-block:: none + Task exception was never retrieved + future: + exception=Exception('not consumed') created at asyncio/tasks.py:321> - Task was destroyed but it is pending! source_traceback: Object created at (most recent call last): - File "test.py", line 15, in - task = asyncio.ensure_future(coro, loop=loop) - task: wait_for= created at test.py:15> - - -.. seealso:: - - :ref:`Detect coroutine objects never scheduled `. - -.. _asyncio-close-transports: + File "../t.py", line 9, in + asyncio.run(main(), debug=True) -Close transports and event loops --------------------------------- + < .. > -When a transport is no more needed, call its ``close()`` method to release -resources. Event loops must also be closed explicitly. - -If a transport or an event loop is not closed explicitly, a -:exc:`ResourceWarning` warning will be emitted in its destructor. By default, -:exc:`ResourceWarning` warnings are ignored. The :ref:`Debug mode of asyncio -` section explains how to display them. + Traceback (most recent call last): + File "../t.py", line 4, in bug + raise Exception("not consumed") + Exception: not consumed diff -Nru python3.7-3.7.0/Doc/library/asyncio-eventloop.rst python3.7-3.7.1/Doc/library/asyncio-eventloop.rst --- python3.7-3.7.0/Doc/library/asyncio-eventloop.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-eventloop.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,103 +1,163 @@ .. currentmodule:: asyncio -.. _asyncio-event-loop: -Base Event Loop -=============== +========== +Event Loop +========== -**Source code:** :source:`Lib/asyncio/events.py` -The event loop is the central execution device provided by :mod:`asyncio`. -It provides multiple facilities, including: +.. rubric:: Preface -* Registering, executing and cancelling delayed calls (timeouts). +The event loop is the core of every asyncio application. +Event loops run asynchronous tasks and callbacks, perform network +IO operations, and run subprocesses. -* Creating client and server :ref:`transports ` for various - kinds of communication. +Application developers should typically use the high-level asyncio functions, +such as :func:`asyncio.run`, and should rarely need to reference the loop +object or call its methods. This section is intended mostly for authors +of lower-level code, libraries, and frameworks, who need finer control over +the event loop behavior. -* Launching subprocesses and the associated :ref:`transports - ` for communication with an external program. +.. rubric:: Obtaining the Event Loop -* Delegating costly function calls to a pool of threads. +The following low-level functions can be used to get, set, or create +an event loop: -.. class:: BaseEventLoop +.. function:: get_running_loop() - This class is an implementation detail. It is a subclass of - :class:`AbstractEventLoop` and may be a base class of concrete - event loop implementations found in :mod:`asyncio`. It should not - be used directly; use :class:`AbstractEventLoop` instead. - ``BaseEventLoop`` should not be subclassed by third-party code; the - internal interface is not stable. + Return the running event loop in the current OS thread. -.. class:: AbstractEventLoop + If there is no running event loop a :exc:`RuntimeError` is raised. + This function can only be called from a coroutine or a callback. - Abstract base class of event loops. + .. versionadded:: 3.7 - This class is :ref:`not thread safe `. +.. function:: get_event_loop() -Run an event loop ------------------ + Get the current event loop. If there is no current event loop set + in the current OS thread and :func:`set_event_loop` has not yet + been called, asyncio will create a new event loop and set it as the + current one. -.. method:: AbstractEventLoop.run_forever() + Because this function has rather complex behavior (especially + when custom event loop policies are in use), using the + :func:`get_running_loop` function is preferred to :func:`get_event_loop` + in coroutines and callbacks. - Run until :meth:`stop` is called. If :meth:`stop` is called before - :meth:`run_forever()` is called, this polls the I/O selector once - with a timeout of zero, runs all callbacks scheduled in response to - I/O events (and those that were already scheduled), and then exits. - If :meth:`stop` is called while :meth:`run_forever` is running, - this will run the current batch of callbacks and then exit. Note - that callbacks scheduled by callbacks will not run in that case; - they will run the next time :meth:`run_forever` is called. + Consider also using the :func:`asyncio.run` function instead of using + lower level functions to manually create and close an event loop. - .. versionchanged:: 3.5.1 +.. function:: set_event_loop(loop) -.. method:: AbstractEventLoop.run_until_complete(future) + Set *loop* as a current event loop for the current OS thread. - Run until the :class:`Future` is done. +.. function:: new_event_loop() - If the argument is a :ref:`coroutine object `, it is wrapped by - :func:`ensure_future`. + Create a new event loop object. - Return the Future's result, or raise its exception. +Note that the behaviour of :func:`get_event_loop`, :func:`set_event_loop`, +and :func:`new_event_loop` functions can be altered by +:ref:`setting a custom event loop policy `. -.. method:: AbstractEventLoop.is_running() - Returns running status of event loop. +.. rubric:: Contents -.. method:: AbstractEventLoop.stop() +This documentation page contains the following sections: - Stop running the event loop. +* The `Event Loop Methods`_ section is the reference documentation of + the event loop APIs; - This causes :meth:`run_forever` to exit at the next suitable - opportunity (see there for more details). +* The `Callback Handles`_ section documents the :class:`Handle` and + :class:`TimerHandle` instances which are returned from scheduling + methods such as :meth:`loop.call_soon` and :meth:`loop.call_later`; - .. versionchanged:: 3.5.1 +* The `Server Objects`_ section documents types returned from + event loop methods like :meth:`loop.create_server`; + +* The `Event Loop Implementations`_ section documents the + :class:`SelectorEventLoop` and :class:`ProactorEventLoop` classes; + +* The `Examples`_ section showcases how to work with some event + loop APIs. + + +.. _asyncio-event-loop: + +Event Loop Methods +================== + +Event loops have **low-level** APIs for the following: + +.. contents:: + :depth: 1 + :local: + + +Running and stopping the loop +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: loop.run_until_complete(future) + + Run until the *future* (an instance of :class:`Future`) has + completed. + + If the argument is a :ref:`coroutine object ` it + is implicitly scheduled to run as a :class:`asyncio.Task`. + + Return the Future's result or raise its exception. + +.. method:: loop.run_forever() + + Run the event loop until :meth:`stop` is called. + + If :meth:`stop` is called before :meth:`run_forever()` is called, + the loop will poll the I/O selector once with a timeout of zero, + run all callbacks scheduled in response to I/O events (and + those that were already scheduled), and then exit. + + If :meth:`stop` is called while :meth:`run_forever` is running, + the loop will run the current batch of callbacks and then exit. + Note that new callbacks scheduled by callbacks will not run in this + case; instead, they will run the next time :meth:`run_forever` or + :meth:`run_until_complete` is called. + +.. method:: loop.stop() + + Stop the event loop. -.. method:: AbstractEventLoop.is_closed() +.. method:: loop.is_running() - Returns ``True`` if the event loop was closed. + Return ``True`` if the event loop is currently running. - .. versionadded:: 3.4.2 +.. method:: loop.is_closed() -.. method:: AbstractEventLoop.close() + Return ``True`` if the event loop was closed. - Close the event loop. The loop must not be running. Pending - callbacks will be lost. +.. method:: loop.close() - This clears the queues and shuts down the executor, but does not wait for - the executor to finish. + Close the event loop. - This is idempotent and irreversible. No other methods should be called after - this one. + The loop must be running when this function is called. + Any pending callbacks will be discarded. + This method clears all queues and shuts down the executor, but does + not wait for the executor to finish. -.. coroutinemethod:: AbstractEventLoop.shutdown_asyncgens() + This method is idempotent and irreversible. No other methods + should be called after the event loop is closed. + +.. coroutinemethod:: loop.shutdown_asyncgens() Schedule all currently open :term:`asynchronous generator` objects to close with an :meth:`~agen.aclose()` call. After calling this method, - the event loop will issue a warning whenever a new asynchronous generator - is iterated. Should be used to finalize all scheduled asynchronous - generators reliably. Example:: + the event loop will issue a warning if a new asynchronous generator + is iterated. This should be used to reliably finalize all scheduled + asynchronous generators. + + Note that there is no need to call this function when + :func:`asyncio.run` is used. + + Example:: try: loop.run_forever() @@ -108,232 +168,223 @@ .. versionadded:: 3.6 -.. _asyncio-pass-keywords: +Scheduling callbacks +^^^^^^^^^^^^^^^^^^^^ -Calls ------ +.. method:: loop.call_soon(callback, *args, context=None) -Most :mod:`asyncio` functions don't accept keywords. If you want to pass -keywords to your callback, use :func:`functools.partial`. For example, -``loop.call_soon(functools.partial(print, "Hello", flush=True))`` will call -``print("Hello", flush=True)``. + Schedule a *callback* to be called with *args* arguments at + the next iteration of the event loop. -.. note:: - :func:`functools.partial` is better than ``lambda`` functions, because - :mod:`asyncio` can inspect :func:`functools.partial` object to display - parameters in debug mode, whereas ``lambda`` functions have a poor - representation. - -.. method:: AbstractEventLoop.call_soon(callback, *args, context=None) - - Arrange for a callback to be called as soon as possible. The callback is - called after :meth:`call_soon` returns, when control returns to the event - loop. - - This operates as a :abbr:`FIFO (first-in, first-out)` queue, callbacks - are called in the order in which they are registered. Each callback - will be called exactly once. - - Any positional arguments after the callback will be passed to the - callback when it is called. - - An optional keyword-only *context* argument allows specifying a custom - :class:`contextvars.Context` for the *callback* to run in. The current - context is used when no *context* is provided. + Callbacks are called in the order in which they are registered. + Each callback will be called exactly once. - An instance of :class:`asyncio.Handle` is returned, which can be - used to cancel the callback. + An optional keyword-only *context* argument allows specifying a + custom :class:`contextvars.Context` for the *callback* to run in. + The current context is used when no *context* is provided. - :ref:`Use functools.partial to pass keywords to the callback - `. + An instance of :class:`asyncio.Handle` is returned, which can be + used later to cancel the callback. - .. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. + This method is not thread-safe. -.. method:: AbstractEventLoop.call_soon_threadsafe(callback, *args, context=None) +.. method:: loop.call_soon_threadsafe(callback, *args, context=None) - Like :meth:`call_soon`, but thread safe. + A thread-safe variant of :meth:`call_soon`. Must be used to + schedule callbacks *from another thread*. See the :ref:`concurrency and multithreading ` section of the documentation. - .. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. +.. versionchanged:: 3.7 + The *context* keyword-only parameter was added. See :pep:`567` + for more details. + +.. _asyncio-pass-keywords: +.. note:: -.. _asyncio-delayed-calls: + Most :mod:`asyncio` scheduling functions don't allow passing + keyword arguments. To do that, use :func:`functools.partial`:: -Delayed calls -------------- + # will schedule "print("Hello", flush=True)" + loop.call_soon( + functools.partial(print, "Hello", flush=True)) + + Using partial objects is usually more convenient than using lambdas, + as asyncio can render partial objects better in debug and error + messages. -The event loop has its own internal clock for computing timeouts. -Which clock is used depends on the (platform-specific) event loop -implementation; ideally it is a monotonic clock. This will generally be -a different clock than :func:`time.time`. -.. note:: +.. _asyncio-delayed-calls: - Timeouts (relative *delay* or absolute *when*) should not exceed one day. +Scheduling delayed callbacks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Event loop provides mechanisms to schedule callback functions +to be called at some point in the future. Event loop uses monotonic +clocks to track time. -.. method:: AbstractEventLoop.call_later(delay, callback, *args, context=None) - Arrange for the *callback* to be called after the given *delay* - seconds (either an int or float). +.. method:: loop.call_later(delay, callback, *args, context=None) - An instance of :class:`asyncio.TimerHandle` is returned, which can be - used to cancel the callback. + Schedule *callback* to be called after the given *delay* + number of seconds (can be either an int or a float). - *callback* will be called exactly once per call to :meth:`call_later`. - If two callbacks are scheduled for exactly the same time, it is - undefined which will be called first. + An instance of :class:`asyncio.TimerHandle` is returned which can + be used to cancel the callback. - The optional positional *args* will be passed to the callback when it - is called. If you want the callback to be called with some named - arguments, use a closure or :func:`functools.partial`. + *callback* will be called exactly once. If two callbacks are + scheduled for exactly the same time, the order in which they + are called is undefined. - An optional keyword-only *context* argument allows specifying a custom - :class:`contextvars.Context` for the *callback* to run in. The current - context is used when no *context* is provided. + The optional positional *args* will be passed to the callback when + it is called. If you want the callback to be called with keyword + arguments use :func:`functools.partial`. - :ref:`Use functools.partial to pass keywords to the callback - `. + An optional keyword-only *context* argument allows specifying a + custom :class:`contextvars.Context` for the *callback* to run in. + The current context is used when no *context* is provided. .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. -.. method:: AbstractEventLoop.call_at(when, callback, *args, context=None) - - Arrange for the *callback* to be called at the given absolute timestamp - *when* (an int or float), using the same time reference as - :meth:`AbstractEventLoop.time`. + .. versionchanged:: 3.7.1 + In Python 3.7.0 and earlier with the default event loop implementation, + the *delay* could not exceed one day. + This has been fixed in Python 3.7.1. + +.. method:: loop.call_at(when, callback, *args, context=None) + + Schedule *callback* to be called at the given absolute timestamp + *when* (an int or a float), using the same time reference as + :meth:`loop.time`. This method's behavior is the same as :meth:`call_later`. - An instance of :class:`asyncio.TimerHandle` is returned, which can be - used to cancel the callback. - - :ref:`Use functools.partial to pass keywords to the callback - `. + An instance of :class:`asyncio.TimerHandle` is returned which can + be used to cancel the callback. .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. -.. method:: AbstractEventLoop.time() + .. versionchanged:: 3.7.1 + In Python 3.7.0 and earlier with the default event loop implementation, + the difference between *when* and the current time could not exceed + one day. This has been fixed in Python 3.7.1. - Return the current time, as a :class:`float` value, according to the - event loop's internal clock. +.. method:: loop.time() -.. seealso:: + Return the current time, as a :class:`float` value, according to + the event loop's internal monotonic clock. - The :func:`asyncio.sleep` function. +.. note:: + Timeouts (relative *delay* or absolute *when*) should not + exceed one day. -Futures -------- +.. seealso:: -.. method:: AbstractEventLoop.create_future() + The :func:`asyncio.sleep` function. - Create an :class:`asyncio.Future` object attached to the loop. - This is a preferred way to create futures in asyncio, as event - loop implementations can provide alternative implementations - of the Future class (with better performance or instrumentation). +Creating Futures and Tasks +^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. versionadded:: 3.5.2 +.. method:: loop.create_future() + Create an :class:`asyncio.Future` object attached to the event loop. -Tasks ------ + This is the preferred way to create Futures in asyncio. This lets + third-party event loops provide alternative implementations of + the Future object (with better performance or instrumentation). -.. method:: AbstractEventLoop.create_task(coro) + .. versionadded:: 3.5.2 - Schedule the execution of a :ref:`coroutine object `: wrap it in - a future. Return a :class:`Task` object. +.. method:: loop.create_task(coro) - Third-party event loops can use their own subclass of :class:`Task` for - interoperability. In this case, the result type is a subclass of - :class:`Task`. + Schedule the execution of a :ref:`coroutine`. + Return a :class:`Task` object. - .. versionadded:: 3.4.2 + Third-party event loops can use their own subclass of :class:`Task` + for interoperability. In this case, the result type is a subclass + of :class:`Task`. -.. method:: AbstractEventLoop.set_task_factory(factory) +.. method:: loop.set_task_factory(factory) Set a task factory that will be used by - :meth:`AbstractEventLoop.create_task`. + :meth:`loop.create_task`. If *factory* is ``None`` the default task factory will be set. + Otherwise, *factory* must be a *callable* with the signature matching + ``(loop, coro)``, where *loop* is a reference to the active + event loop, and *coro* is a coroutine object. The callable + must return a :class:`asyncio.Future`-compatible object. - If *factory* is a *callable*, it should have a signature matching - ``(loop, coro)``, where *loop* will be a reference to the active - event loop, *coro* will be a coroutine object. The callable - must return an :class:`asyncio.Future` compatible object. +.. method:: loop.get_task_factory() - .. versionadded:: 3.4.4 + Return a task factory or ``None`` if the default one is in use. -.. method:: AbstractEventLoop.get_task_factory() - Return a task factory, or ``None`` if the default one is in use. +Opening network connections +^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. versionadded:: 3.4.4 +.. coroutinemethod:: loop.create_connection(protocol_factory, \ + host=None, port=None, \*, ssl=None, \ + family=0, proto=0, flags=0, sock=None, \ + local_addr=None, server_hostname=None, \ + ssl_handshake_timeout=None) + Open a streaming transport connection to a given + address specified by *host* and *port*. -Creating connections --------------------- + The socket family can be either :py:data:`~socket.AF_INET` or + :py:data:`~socket.AF_INET6` depending on *host* (or the *family* + argument, if provided). -.. coroutinemethod:: AbstractEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None, ssl_handshake_timeout=None) + The socket type will be :py:data:`~socket.SOCK_STREAM`. - Create a streaming transport connection to a given Internet *host* and - *port*: socket family :py:data:`~socket.AF_INET` or - :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified), - socket type :py:data:`~socket.SOCK_STREAM`. *protocol_factory* must be a - callable returning a :ref:`protocol ` instance. + *protocol_factory* must be a callable returning an + :ref:`asyncio protocol ` implementation. This method will try to establish the connection in the background. When successful, it returns a ``(transport, protocol)`` pair. The chronological synopsis of the underlying operation is as follows: - #. The connection is established, and a :ref:`transport ` - is created to represent it. + #. The connection is established and a :ref:`transport ` + is created for it. - #. *protocol_factory* is called without arguments and must return a - :ref:`protocol ` instance. + #. *protocol_factory* is called without arguments and is expected to + return a :ref:`protocol ` instance. - #. The protocol instance is tied to the transport, and its - :meth:`connection_made` method is called. + #. The protocol instance is coupled with the transport by calling its + :meth:`~BaseProtocol.connection_made` method. - #. The coroutine returns successfully with the ``(transport, protocol)`` - pair. + #. A ``(transport, protocol)`` tuple is returned on success. - The created transport is an implementation-dependent bidirectional stream. + The created transport is an implementation-dependent bidirectional + stream. - .. note:: - *protocol_factory* can be any kind of callable, not necessarily - a class. For example, if you want to use a pre-created - protocol instance, you can pass ``lambda: my_protocol``. - - Options that change how the connection is created: + Other arguments: * *ssl*: if given and not false, a SSL/TLS transport is created (by default a plain TCP transport is created). If *ssl* is a :class:`ssl.SSLContext` object, this context is used to create - the transport; if *ssl* is :const:`True`, a context with some - unspecified default settings is used. + the transport; if *ssl* is :const:`True`, a default context returned + from :func:`ssl.create_default_context` is used. .. seealso:: :ref:`SSL/TLS security considerations ` - * *server_hostname*, is only for use together with *ssl*, - and sets or overrides the hostname that the target server's certificate - will be matched against. By default the value of the *host* argument + * *server_hostname* sets or overrides the hostname that the target + server's certificate will be matched against. Should only be passed + if *ssl* is not ``None``. By default the value of the *host* argument is used. If *host* is empty, there is no default and you must pass a value for *server_hostname*. If *server_hostname* is an empty string, hostname matching is disabled (which is a serious security - risk, allowing for man-in-the-middle-attacks). + risk, allowing for potential man-in-the-middle attacks). * *family*, *proto*, *flags* are the optional address family, protocol and flags to be passed through to getaddrinfo() for *host* resolution. @@ -347,38 +398,51 @@ * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used to bind the socket to locally. The *local_host* and *local_port* - are looked up using getaddrinfo(), similarly to *host* and *port*. + are looked up using ``getaddrinfo()``, similarly to *host* and *port*. - * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds - to wait for the SSL handshake to complete before aborting the connection. + * *ssl_handshake_timeout* is (for a TLS connection) the time in seconds + to wait for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). .. versionadded:: 3.7 The *ssl_handshake_timeout* parameter. + .. versionchanged:: 3.6 + + The socket option :py:data:`~socket.TCP_NODELAY` is set by default + for all TCP connections. + .. versionchanged:: 3.5 - On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported. + Added support for SSL/TLS in :class:`ProactorEventLoop`. .. seealso:: - The :func:`open_connection` function can be used to get a pair of - (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol. + The :func:`open_connection` function is a high-level alternative + API. It returns a pair of (:class:`StreamReader`, :class:`StreamWriter`) + that can be used directly in async/await code. +.. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \ + local_addr=None, remote_addr=None, \*, \ + family=0, proto=0, flags=0, \ + reuse_address=None, reuse_port=None, \ + allow_broadcast=None, sock=None) -.. coroutinemethod:: AbstractEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None) + Create a datagram connection. - Create datagram connection: socket family :py:data:`~socket.AF_INET`, - :py:data:`~socket.AF_INET6` or :py:data:`~socket.AF_UNIX` depending on - *host* (or *family* if specified), socket type - :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a - callable returning a :ref:`protocol ` instance. + The socket family can be either :py:data:`~socket.AF_INET`, + :py:data:`~socket.AF_INET6`, or :py:data:`~socket.AF_UNIX`, + depending on *host* (or the *family* argument, if provided). - This method will try to establish the connection in the background. - When successful, it returns a ``(transport, protocol)`` pair. + The socket type will be :py:data:`~socket.SOCK_DGRAM`. - Options changing how the connection is created: + *protocol_factory* must be a callable returning a + :ref:`protocol ` implementation. + + A tuple of ``(transport, protocol)`` is returned on success. + + Other arguments: * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used to bind the socket to locally. The *local_host* and *local_port* @@ -394,14 +458,14 @@ corresponding :mod:`socket` module constants. * *reuse_address* tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to + ``TIME_WAIT`` state, without waiting for its natural timeout to expire. If not specified will automatically be set to ``True`` on - UNIX. + Unix. * *reuse_port* tells the kernel to allow this endpoint to be bound to the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows - and some UNIX's. If the :py:data:`~socket.SO_REUSEPORT` constant is not + and some Unixes. If the :py:data:`~socket.SO_REUSEPORT` constant is not defined then this capability is unsupported. * *allow_broadcast* tells the kernel to allow this endpoint to send @@ -412,7 +476,7 @@ transport. If specified, *local_addr* and *remote_addr* should be omitted (must be :const:`None`). - On Windows with :class:`ProactorEventLoop`, this method is not supported. + On Windows, with :class:`ProactorEventLoop`, this method is not supported. See :ref:`UDP echo client protocol ` and :ref:`UDP echo server protocol ` examples. @@ -421,23 +485,26 @@ The *family*, *proto*, *flags*, *reuse_address*, *reuse_port, *allow_broadcast*, and *sock* parameters were added. -.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path=None, \*, ssl=None, sock=None, server_hostname=None, ssl_handshake_timeout=None) +.. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ + path=None, \*, ssl=None, sock=None, \ + server_hostname=None, ssl_handshake_timeout=None) - Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket - type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket - family is used to communicate between processes on the same machine - efficiently. + Create a Unix connection. - This method will try to establish the connection in the background. - When successful, it returns a ``(transport, protocol)`` pair. + The socket family will be :py:data:`~socket.AF_UNIX`; socket + type will be :py:data:`~socket.SOCK_STREAM`. + + A tuple of ``(transport, protocol)`` is returned on success. - *path* is the name of a UNIX domain socket, and is required unless a *sock* - parameter is specified. Abstract UNIX sockets, :class:`str`, - :class:`bytes`, and :class:`~pathlib.Path` paths are supported. + *path* is the name of a Unix domain socket and is required, + unless a *sock* parameter is specified. Abstract Unix sockets, + :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are + supported. - See the :meth:`AbstractEventLoop.create_connection` method for parameters. + See the documentation of the :meth:`loop.create_connection` method + for information about arguments to this method. - Availability: UNIX. + Availability: Unix. .. versionadded:: 3.7 @@ -448,55 +515,68 @@ The *path* parameter can now be a :term:`path-like object`. -Creating listening connections ------------------------------- +Creating network servers +^^^^^^^^^^^^^^^^^^^^^^^^ -.. coroutinemethod:: AbstractEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, ssl_handshake_timeout=None, start_serving=True) +.. coroutinemethod:: loop.create_server(protocol_factory, \ + host=None, port=None, \*, \ + family=socket.AF_UNSPEC, \ + flags=socket.AI_PASSIVE, \ + sock=None, backlog=100, ssl=None, \ + reuse_address=None, reuse_port=None, \ + ssl_handshake_timeout=None, start_serving=True) - Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to - *host* and *port*. + Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) listening + on *port* of the *host* address. - Return a :class:`Server` object, its :attr:`~Server.sockets` attribute - contains created sockets. Use the :meth:`Server.close` method to stop the - server: close listening sockets. + Returns a :class:`Server` object. - Parameters: + Arguments: + + * *protocol_factory* must be a callable returning a + :ref:`protocol ` implementation. + + * The *host* parameter can be set to several types which determine where + the server would be listening: + + - If *host* is a string, the TCP server is bound to a single network + interface specified by *host*. - * The *host* parameter can be a string, in that case the TCP server is - bound to *host* and *port*. The *host* parameter can also be a sequence - of strings and in that case the TCP server is bound to all hosts of the - sequence. If *host* is an empty string or ``None``, all interfaces are - assumed and a list of multiple sockets will be returned (most likely one - for IPv4 and another one for IPv6). + - If *host* is a sequence of strings, the TCP server is bound to all + network interfaces specified by the sequence. + + - If *host* is an empty string or ``None``, all interfaces are + assumed and a list of multiple sockets will be returned (most likely + one for IPv4 and another one for IPv6). * *family* can be set to either :data:`socket.AF_INET` or - :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set - it will be determined from host (defaults to :data:`socket.AF_UNSPEC`). + :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. + If not set, the *family* will be determined from host name + (defaults to :data:`~socket.AF_UNSPEC`). * *flags* is a bitmask for :meth:`getaddrinfo`. * *sock* can optionally be specified in order to use a preexisting - socket object. If specified, *host* and *port* should be omitted (must be - :const:`None`). + socket object. If specified, *host* and *port* must not be specified. * *backlog* is the maximum number of queued connections passed to :meth:`~socket.socket.listen` (defaults to 100). - * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the - accepted connections. + * *ssl* can be set to an :class:`~ssl.SSLContext` instance to enable + TLS over the accepted connections. * *reuse_address* tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to + ``TIME_WAIT`` state, without waiting for its natural timeout to expire. If not specified will automatically be set to ``True`` on - UNIX. + Unix. * *reuse_port* tells the kernel to allow this endpoint to be bound to the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows. - * *ssl_handshake_timeout* is (for an SSL server) the time in seconds to wait - for the SSL handshake to complete before aborting the connection. + * *ssl_handshake_timeout* is (for a TLS server) the time in seconds to wait + for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). * *start_serving* set to ``True`` (the default) causes the created server @@ -507,32 +587,44 @@ .. versionadded:: 3.7 - *ssl_handshake_timeout* and *start_serving* parameters. + Added *ssl_handshake_timeout* and *start_serving* parameters. - .. versionchanged:: 3.5 + .. versionchanged:: 3.6 - On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported. + The socket option :py:data:`~socket.TCP_NODELAY` is set by default + for all TCP connections. - .. seealso:: + .. versionchanged:: 3.5 - The function :func:`start_server` creates a (:class:`StreamReader`, - :class:`StreamWriter`) pair and calls back a function with this pair. + Added support for SSL/TLS in :class:`ProactorEventLoop`. .. versionchanged:: 3.5.1 - The *host* parameter can now be a sequence of strings. + The *host* parameter can be a sequence of strings. + + .. seealso:: + + The :func:`start_server` function is a higher-level alternative API + that returns a pair of :class:`StreamReader` and :class:`StreamWriter` + that can be used in an async/await code. + +.. coroutinemethod:: loop.create_unix_server(protocol_factory, path=None, \ + \*, sock=None, backlog=100, ssl=None, \ + ssl_handshake_timeout=None, start_serving=True) -.. coroutinemethod:: AbstractEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, start_serving=True) + Similar to :meth:`loop.create_server` but works with the + :py:data:`~socket.AF_UNIX` socket family. - Similar to :meth:`AbstractEventLoop.create_server`, but specific to the - socket family :py:data:`~socket.AF_UNIX`. + *path* is the name of a Unix domain socket, and is required, + unless a *sock* argument is provided. Abstract Unix sockets, + :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths + are supported. - *path* is the name of a UNIX domain socket, and is required unless a *sock* - parameter is specified. Abstract UNIX sockets, :class:`str`, - :class:`bytes`, and :class:`~pathlib.Path` paths are supported. + See the documentation of the :meth:`loop.create_server` method + for information about arguments to this method. - Availability: UNIX. + Availability: Unix. .. versionadded:: 3.7 @@ -542,26 +634,30 @@ The *path* parameter can now be a :class:`~pathlib.Path` object. -.. coroutinemethod:: BaseEventLoop.connect_accepted_socket(protocol_factory, sock, \*, ssl=None, ssl_handshake_timeout=None) +.. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ + sock, \*, ssl=None, ssl_handshake_timeout=None) - Handle an accepted connection. + Wrap an already accepted connection into a transport/protocol pair. - This is used by servers that accept connections outside of - asyncio but that use asyncio to handle them. + This method can be used by servers that accept connections outside + of asyncio but that use asyncio to handle them. Parameters: - * *sock* is a preexisting socket object returned from an ``accept`` - call. + * *protocol_factory* must be a callable returning a + :ref:`protocol ` implementation. + + * *sock* is a preexisting socket object returned from + :meth:`socket.accept `. - * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the - accepted connections. + * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over + the accepted connections. * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to wait for the SSL handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). - When completed it returns a ``(transport, protocol)`` pair. + Returns a ``(transport, protocol)`` pair. .. versionadded:: 3.7 @@ -570,15 +666,14 @@ .. versionadded:: 3.5.3 -File Transferring ------------------ +Transferring files +^^^^^^^^^^^^^^^^^^ -.. coroutinemethod:: AbstractEventLoop.sendfile(transport, file, \ - offset=0, count=None, \ - *, fallback=True) +.. coroutinemethod:: loop.sendfile(transport, file, \ + offset=0, count=None, *, fallback=True) - Send a *file* to *transport*, return the total number of bytes - which were sent. + Send a *file* over a *transport*. Return the total number of bytes + sent. The method uses high-performance :meth:`os.sendfile` if available. @@ -586,167 +681,163 @@ *offset* tells from where to start reading the file. If specified, *count* is the total number of bytes to transmit as opposed to - sending the file until EOF is reached. File position is updated on - return or also in case of error in which case :meth:`file.tell() - ` can be used to figure out the number of bytes - which were sent. + sending the file until EOF is reached. File position is always updated, + even when this method raises an error, and + :meth:`file.tell() ` can be used to obtain the actual + number of bytes sent. *fallback* set to ``True`` makes asyncio to manually read and send - the file when the platform does not support the sendfile syscall + the file when the platform does not support the sendfile system call (e.g. Windows or SSL socket on Unix). Raise :exc:`SendfileNotAvailableError` if the system does not support - *sendfile* syscall and *fallback* is ``False``. + the *sendfile* syscall and *fallback* is ``False``. .. versionadded:: 3.7 TLS Upgrade ------------ +^^^^^^^^^^^ -.. coroutinemethod:: AbstractEventLoop.start_tls(transport, protocol, sslcontext, \*, server_side=False, server_hostname=None, ssl_handshake_timeout=None) +.. coroutinemethod:: loop.start_tls(transport, protocol, \ + sslcontext, \*, server_side=False, \ + server_hostname=None, ssl_handshake_timeout=None) - Upgrades an existing connection to TLS. + Upgrade an existing transport-based connection to TLS. - Returns a new transport instance, that the *protocol* must start using + Return a new transport instance, that the *protocol* must start using immediately after the *await*. The *transport* instance passed to the *start_tls* method should never be used again. Parameters: * *transport* and *protocol* instances that methods like - :meth:`~AbstractEventLoop.create_server` and - :meth:`~AbstractEventLoop.create_connection` return. + :meth:`~loop.create_server` and + :meth:`~loop.create_connection` return. * *sslcontext*: a configured instance of :class:`~ssl.SSLContext`. * *server_side* pass ``True`` when a server-side connection is being - upgraded (like the one created by :meth:`~AbstractEventLoop.create_server`). + upgraded (like the one created by :meth:`~loop.create_server`). * *server_hostname*: sets or overrides the host name that the target server's certificate will be matched against. - * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to - wait for the SSL handshake to complete before aborting the connection. + * *ssl_handshake_timeout* is (for a TLS connection) the time in seconds to + wait for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). .. versionadded:: 3.7 -Watch file descriptors ----------------------- - -On Windows with :class:`SelectorEventLoop`, only socket handles are supported -(ex: pipe file descriptors are not supported). - -On Windows with :class:`ProactorEventLoop`, these methods are not supported. +Watching file descriptors +^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: AbstractEventLoop.add_reader(fd, callback, \*args) +.. method:: loop.add_reader(fd, callback, \*args) - Start watching the file descriptor for read availability and then call the - *callback* with specified arguments. + Start monitoring the *fd* file descriptor for read availability and + invoke *callback* with the specified arguments once *fd* is available for + reading. - :ref:`Use functools.partial to pass keywords to the callback - `. +.. method:: loop.remove_reader(fd) -.. method:: AbstractEventLoop.remove_reader(fd) + Stop monitoring the *fd* file descriptor for read availability. - Stop watching the file descriptor for read availability. +.. method:: loop.add_writer(fd, callback, \*args) -.. method:: AbstractEventLoop.add_writer(fd, callback, \*args) + Start monitoring the *fd* file descriptor for write availability and + invoke *callback* with the specified arguments once *fd* is available for + writing. - Start watching the file descriptor for write availability and then call the - *callback* with specified arguments. + Use :func:`functools.partial` :ref:`to pass keyword arguments + ` to *func*. - :ref:`Use functools.partial to pass keywords to the callback - `. +.. method:: loop.remove_writer(fd) -.. method:: AbstractEventLoop.remove_writer(fd) + Stop monitoring the *fd* file descriptor for write availability. - Stop watching the file descriptor for write availability. +See also :ref:`Platform Support ` section +for some limitations of these methods. -The :ref:`watch a file descriptor for read events ` -example uses the low-level :meth:`AbstractEventLoop.add_reader` method to register -the file descriptor of a socket. +Working with socket objects directly +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Low-level socket operations ---------------------------- +In general, protocol implementations that use transport-based APIs +such as :meth:`loop.create_connection` and :meth:`loop.create_server` +are faster than implementations that work with sockets directly. +However, there are some use cases when performance is not critical, and +working with :class:`~socket.socket` objects directly is more +convenient. -.. coroutinemethod:: AbstractEventLoop.sock_recv(sock, nbytes) +.. coroutinemethod:: loop.sock_recv(sock, nbytes) - Receive data from the socket. Modeled after blocking - :meth:`socket.socket.recv` method. + Receive up to *nbytes* from *sock*. Asynchronous version of + :meth:`socket.recv() `. - The return value is a bytes object - representing the data received. The maximum amount of data to be received - at once is specified by *nbytes*. + Return the received data as a bytes object. - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. + *sock* must be a non-blocking socket. .. versionchanged:: 3.7 - Even though the method was always documented as a coroutine - method, before Python 3.7 it returned a :class:`Future`. - Since Python 3.7, this is an ``async def`` method. + Even though this method was always documented as a coroutine + method, releases before Python 3.7 returned a :class:`Future`. + Since Python 3.7 this is an ``async def`` method. -.. coroutinemethod:: AbstractEventLoop.sock_recv_into(sock, buf) +.. coroutinemethod:: loop.sock_recv_into(sock, buf) - Receive data from the socket. Modeled after blocking - :meth:`socket.socket.recv_into` method. + Receive data from *sock* into the *buf* buffer. Modeled after the blocking + :meth:`socket.recv_into() ` method. - The received data is written into *buf* (a writable buffer). - The return value is the number of bytes written. + Return the number of bytes written to the buffer. - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. + *sock* must be a non-blocking socket. .. versionadded:: 3.7 -.. coroutinemethod:: AbstractEventLoop.sock_sendall(sock, data) +.. coroutinemethod:: loop.sock_sendall(sock, data) - Send data to the socket. Modeled after blocking - :meth:`socket.socket.sendall` method. + Send *data* to the *sock* socket. Asynchronous version of + :meth:`socket.sendall() `. - The socket must be connected to a remote socket. - This method continues to send data from *data* until either all data has - been sent or an error occurs. ``None`` is returned on success. On error, - an exception is raised, and there is no way to determine how much data, if - any, was successfully processed by the receiving end of the connection. + This method continues to send to the socket until either all data + in *data* has been sent or an error occurs. ``None`` is returned + on success. On error, an exception is raised. Additionally, there is no way + to determine how much data, if any, was successfully processed by the + receiving end of the connection. - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. + *sock* must be a non-blocking socket. .. versionchanged:: 3.7 Even though the method was always documented as a coroutine method, before Python 3.7 it returned an :class:`Future`. Since Python 3.7, this is an ``async def`` method. -.. coroutinemethod:: AbstractEventLoop.sock_connect(sock, address) +.. coroutinemethod:: loop.sock_connect(sock, address) - Connect to a remote socket at *address*. Modeled after - blocking :meth:`socket.socket.connect` method. + Connect *sock* to a remote socket at *address*. - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. + Asynchronous version of :meth:`socket.connect() `. + + *sock* must be a non-blocking socket. .. versionchanged:: 3.5.2 ``address`` no longer needs to be resolved. ``sock_connect`` will try to check if the *address* is already resolved by calling :func:`socket.inet_pton`. If not, - :meth:`AbstractEventLoop.getaddrinfo` will be used to resolve the + :meth:`loop.getaddrinfo` will be used to resolve the *address*. .. seealso:: - :meth:`AbstractEventLoop.create_connection` + :meth:`loop.create_connection` and :func:`asyncio.open_connection() `. -.. coroutinemethod:: AbstractEventLoop.sock_accept(sock) +.. coroutinemethod:: loop.sock_accept(sock) - Accept a connection. Modeled after blocking - :meth:`socket.socket.accept`. + Accept a connection. Modeled after the blocking + :meth:`socket.accept() ` method. The socket must be bound to an address and listening for connections. The return value is a pair ``(conn, address)`` where *conn* @@ -754,7 +845,7 @@ and *address* is the address bound to the socket on the other end of the connection. - The socket *sock* must be non-blocking. + *sock* must be a non-blocking socket. .. versionchanged:: 3.7 Even though the method was always documented as a coroutine @@ -763,51 +854,51 @@ .. seealso:: - :meth:`AbstractEventLoop.create_server` and :func:`start_server`. + :meth:`loop.create_server` and :func:`start_server`. -.. coroutinemethod:: AbstractEventLoop.sock_sendfile(sock, file, \ - offset=0, count=None, \ - *, fallback=True) +.. coroutinemethod:: loop.sock_sendfile(sock, file, offset=0, count=None, \ + \*, fallback=True) - Send a file using high-performance :mod:`os.sendfile` if possible - and return the total number of bytes which were sent. + Send a file using high-performance :mod:`os.sendfile` if possible. + Return the total number of bytes sent. - Asynchronous version of :meth:`socket.socket.sendfile`. + Asynchronous version of :meth:`socket.sendfile() `. - *sock* must be non-blocking :class:`~socket.socket` of - :const:`socket.SOCK_STREAM` type. + *sock* must be a non-blocking :const:`socket.SOCK_STREAM` + :class:`~socket.socket`. - *file* must be a regular file object opened in binary mode. + *file* must be a regular file object open in binary mode. *offset* tells from where to start reading the file. If specified, *count* is the total number of bytes to transmit as opposed to - sending the file until EOF is reached. File position is updated on - return or also in case of error in which case :meth:`file.tell() - ` can be used to figure out the number of bytes - which were sent. + sending the file until EOF is reached. File position is always updated, + even when this method raises an error, and + :meth:`file.tell() ` can be used to obtain the actual + number of bytes sent. - *fallback* set to ``True`` makes asyncio to manually read and send + *fallback*, when set to ``True``, makes asyncio manually read and send the file when the platform does not support the sendfile syscall (e.g. Windows or SSL socket on Unix). Raise :exc:`SendfileNotAvailableError` if the system does not support *sendfile* syscall and *fallback* is ``False``. + *sock* must be a non-blocking socket. + .. versionadded:: 3.7 -Resolve host name ------------------ +DNS +^^^ -.. coroutinemethod:: AbstractEventLoop.getaddrinfo(host, port, \*, family=0, type=0, proto=0, flags=0) +.. coroutinemethod:: loop.getaddrinfo(host, port, \*, family=0, \ + type=0, proto=0, flags=0) - This method is a :ref:`coroutine `, similar to - :meth:`socket.getaddrinfo` function but non-blocking. + Asynchronous version of :meth:`socket.getaddrinfo`. -.. coroutinemethod:: AbstractEventLoop.getnameinfo(sockaddr, flags=0) +.. coroutinemethod:: loop.getnameinfo(sockaddr, flags=0) - This method is a :ref:`coroutine `, similar to - :meth:`socket.getnameinfo` function but non-blocking. + Asynchronous version of :meth:`socket.getnameinfo`. .. versionchanged:: 3.7 Both *getaddrinfo* and *getnameinfo* methods were always documented @@ -816,141 +907,198 @@ both methods are coroutines. -Connect pipes -------------- +Working with pipes +^^^^^^^^^^^^^^^^^^ + +.. coroutinemethod:: loop.connect_read_pipe(protocol_factory, pipe) -On Windows with :class:`SelectorEventLoop`, these methods are not supported. -Use :class:`ProactorEventLoop` to support pipes on Windows. + Register the read end of *pipe* in the event loop. -.. coroutinemethod:: AbstractEventLoop.connect_read_pipe(protocol_factory, pipe) + *protocol_factory* must be a callable returning an + :ref:`asyncio protocol ` implementation. - Register read pipe in eventloop. + *pipe* is a :term:`file-like object `. - *protocol_factory* should instantiate object with :class:`Protocol` - interface. *pipe* is a :term:`file-like object `. - Return pair ``(transport, protocol)``, where *transport* supports the - :class:`ReadTransport` interface. + Return pair ``(transport, protocol)``, where *transport* supports + the :class:`ReadTransport` interface and *protocol* is an object + instantiated by the *protocol_factory*. With :class:`SelectorEventLoop` event loop, the *pipe* is set to non-blocking mode. -.. coroutinemethod:: AbstractEventLoop.connect_write_pipe(protocol_factory, pipe) +.. coroutinemethod:: loop.connect_write_pipe(protocol_factory, pipe) - Register write pipe in eventloop. + Register the write end of *pipe* in the event loop. + + *protocol_factory* must be a callable returning an + :ref:`asyncio protocol ` implementation. + + *pipe* is :term:`file-like object `. - *protocol_factory* should instantiate object with :class:`BaseProtocol` - interface. *pipe* is :term:`file-like object `. Return pair ``(transport, protocol)``, where *transport* supports - :class:`WriteTransport` interface. + :class:`WriteTransport` interface and *protocol* is an object + instantiated by the *protocol_factory*. With :class:`SelectorEventLoop` event loop, the *pipe* is set to non-blocking mode. -.. seealso:: +.. note:: + + :class:`SelectorEventLoop` does not support the above methods on + Windows. Use :class:`ProactorEventLoop` instead for Windows. - The :meth:`AbstractEventLoop.subprocess_exec` and - :meth:`AbstractEventLoop.subprocess_shell` methods. +.. seealso:: + The :meth:`loop.subprocess_exec` and + :meth:`loop.subprocess_shell` methods. -UNIX signals ------------- -Availability: UNIX only. +Unix signals +^^^^^^^^^^^^ -.. method:: AbstractEventLoop.add_signal_handler(signum, callback, \*args) +.. method:: loop.add_signal_handler(signum, callback, \*args) - Add a handler for a signal. + Set *callback* as the handler for the *signum* signal. Raise :exc:`ValueError` if the signal number is invalid or uncatchable. Raise :exc:`RuntimeError` if there is a problem setting up the handler. - :ref:`Use functools.partial to pass keywords to the callback - `. + Use :func:`functools.partial` :ref:`to pass keyword arguments + ` to *func*. -.. method:: AbstractEventLoop.remove_signal_handler(sig) +.. method:: loop.remove_signal_handler(sig) - Remove a handler for a signal. + Remove the handler for the *sig* signal. - Return ``True`` if a signal handler was removed, ``False`` if not. + Return ``True`` if the signal handler was removed, or ``False`` if + no handler was set for the given signal. + +Availability: Unix. .. seealso:: The :mod:`signal` module. -Executor --------- - -Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or -pool of processes). By default, an event loop uses a thread pool executor -(:class:`~concurrent.futures.ThreadPoolExecutor`). +Executing code in thread or process pools +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: AbstractEventLoop.run_in_executor(executor, func, \*args) +.. awaitablemethod:: loop.run_in_executor(executor, func, \*args) - Arrange for a *func* to be called in the specified executor. + Arrange for *func* to be called in the specified executor. - The *executor* argument should be an :class:`~concurrent.futures.Executor` + The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. - :ref:`Use functools.partial to pass keywords to the *func* - `. + Example:: + + import asyncio + import concurrent.futures + + def blocking_io(): + # File operations (such as logging) can block the + # event loop: run them in a thread pool. + with open('/dev/urandom', 'rb') as f: + return f.read(100) + + def cpu_bound(): + # CPU-bound operations will block the event loop: + # in general it is preferable to run them in a + # process pool. + return sum(i * i for i in range(10 ** 7)) + + async def main(): + loop = asyncio.get_running_loop() + + ## Options: + + # 1. Run in the default loop's executor: + result = await loop.run_in_executor( + None, blocking_io) + print('default thread pool', result) + + # 2. Run in a custom thread pool: + with concurrent.futures.ThreadPoolExecutor() as pool: + result = await loop.run_in_executor( + pool, blocking_io) + print('custom thread pool', result) + + # 3. Run in a custom process pool: + with concurrent.futures.ProcessPoolExecutor() as pool: + result = await loop.run_in_executor( + pool, cpu_bound) + print('custom process pool', result) + + asyncio.run(main()) This method returns a :class:`asyncio.Future` object. + Use :func:`functools.partial` :ref:`to pass keyword arguments + ` to *func*. + .. versionchanged:: 3.5.3 - :meth:`BaseEventLoop.run_in_executor` no longer configures the + :meth:`loop.run_in_executor` no longer configures the ``max_workers`` of the thread pool executor it creates, instead leaving it up to the thread pool executor (:class:`~concurrent.futures.ThreadPoolExecutor`) to set the default. -.. method:: AbstractEventLoop.set_default_executor(executor) +.. method:: loop.set_default_executor(executor) - Set the default executor used by :meth:`run_in_executor`. + Set *executor* as the default executor used by :meth:`run_in_executor`. + *executor* should be an instance of + :class:`~concurrent.futures.ThreadPoolExecutor`. + + .. deprecated:: 3.7 + Using an executor that is not an instance of + :class:`~concurrent.futures.ThreadPoolExecutor` is deprecated and + will trigger an error in Python 3.9. + + *executor* must be an instance of + :class:`concurrent.futures.ThreadPoolExecutor`. Error Handling API ------------------- +^^^^^^^^^^^^^^^^^^ Allows customizing how exceptions are handled in the event loop. -.. method:: AbstractEventLoop.set_exception_handler(handler) +.. method:: loop.set_exception_handler(handler) Set *handler* as the new event loop exception handler. If *handler* is ``None``, the default exception handler will - be set. - - If *handler* is a callable object, it should have a - matching signature to ``(loop, context)``, where ``loop`` - will be a reference to the active event loop, ``context`` - will be a ``dict`` object (see :meth:`call_exception_handler` - documentation for details about context). + be set. Otherwise, *handler* must be a callable with the signature + matching ``(loop, context)``, where ``loop`` + is a reference to the active event loop, and ``context`` + is a ``dict`` object containing the details of the exception + (see :meth:`call_exception_handler` documentation for details + about context). -.. method:: AbstractEventLoop.get_exception_handler() +.. method:: loop.get_exception_handler() - Return the exception handler, or ``None`` if the default one - is in use. + Return the current exception handler, or ``None`` if no custom + exception handler was set. .. versionadded:: 3.5.2 -.. method:: AbstractEventLoop.default_exception_handler(context) +.. method:: loop.default_exception_handler(context) Default exception handler. This is called when an exception occurs and no exception - handler is set, and can be called by a custom exception - handler that wants to defer to the default behavior. + handler is set. This can be called by a custom exception + handler that wants to defer to the default handler behavior. *context* parameter has the same meaning as in :meth:`call_exception_handler`. -.. method:: AbstractEventLoop.call_exception_handler(context) +.. method:: loop.call_exception_handler(context) Call the current event loop exception handler. *context* is a ``dict`` object containing the following keys - (new keys may be introduced later): + (new keys may be introduced in future Python versions): * 'message': Error message; * 'exception' (optional): Exception object; @@ -962,14 +1110,14 @@ .. note:: - Note: this method should not be overloaded in subclassed - event loops. For any custom exception handling, use - :meth:`set_exception_handler()` method. + This method should not be overloaded in subclassed + event loops. For custom exception handling, use + the :meth:`set_exception_handler()` method. -Debug mode ----------- +Enabling debug mode +^^^^^^^^^^^^^^^^^^^ -.. method:: AbstractEventLoop.get_debug() +.. method:: loop.get_debug() Get the debug mode (:class:`bool`) of the event loop. @@ -977,29 +1125,173 @@ :envvar:`PYTHONASYNCIODEBUG` is set to a non-empty string, ``False`` otherwise. - .. versionadded:: 3.4.2 - -.. method:: AbstractEventLoop.set_debug(enabled: bool) +.. method:: loop.set_debug(enabled: bool) Set the debug mode of the event loop. - .. versionadded:: 3.4.2 + .. versionchanged:: 3.7 + + The new ``-X dev`` command line option can now also be used + to enable the debug mode. .. seealso:: The :ref:`debug mode of asyncio `. -Server ------- -.. class:: Server +Running Subprocesses +^^^^^^^^^^^^^^^^^^^^ + +Methods described in this subsections are low-level. In regular +async/await code consider using the high-level +:func:`asyncio.create_subprocess_shell` and +:func:`asyncio.create_subprocess_exec` convenience functions instead. + +.. note:: + + The default asyncio event loop on **Windows** does not support + subprocesses. See :ref:`Subprocess Support on Windows + ` for details. + +.. coroutinemethod:: loop.subprocess_exec(protocol_factory, \*args, \ + stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, \*\*kwargs) + + Create a subprocess from one or more string arguments specified by + *args*. + + *args* must be a list of strings represented by: + + * :class:`str`; + * or :class:`bytes`, encoded to the + :ref:`filesystem encoding `. + + The first string specifies the program executable, + and the remaining strings specify the arguments. Together, string + arguments form the ``argv`` of the program. + + This is similar to the standard library :class:`subprocess.Popen` + class called with ``shell=False`` and the list of strings passed as + the first argument; however, where :class:`~subprocess.Popen` takes + a single argument which is list of strings, *subprocess_exec* + takes multiple string arguments. + + The *protocol_factory* must be a callable returning a subclass of the + :class:`asyncio.SubprocessProtocol` class. + + Other parameters: + + * *stdin*: either a file-like object representing a pipe to be + connected to the subprocess's standard input stream using + :meth:`~loop.connect_write_pipe`, or the + :const:`subprocess.PIPE` constant (default). By default a new + pipe will be created and connected. + + * *stdout*: either a file-like object representing the pipe to be + connected to the subprocess's standard output stream using + :meth:`~loop.connect_read_pipe`, or the + :const:`subprocess.PIPE` constant (default). By default a new pipe + will be created and connected. + + * *stderr*: either a file-like object representing the pipe to be + connected to the subprocess's standard error stream using + :meth:`~loop.connect_read_pipe`, or one of + :const:`subprocess.PIPE` (default) or :const:`subprocess.STDOUT` + constants. + + By default a new pipe will be created and connected. When + :const:`subprocess.STDOUT` is specified, the subprocess' standard + error stream will be connected to the same pipe as the standard + output stream. + + * All other keyword arguments are passed to :class:`subprocess.Popen` + without interpretation, except for *bufsize*, *universal_newlines* + and *shell*, which should not be specified at all. + + See the constructor of the :class:`subprocess.Popen` class + for documentation on other arguments. + + Returns a pair of ``(transport, protocol)``, where *transport* + conforms to the :class:`asyncio.SubprocessTransport` base class and + *protocol* is an object instantiated by the *protocol_factory*. + +.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, \*, \ + stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, \*\*kwargs) + + Create a subprocess from *cmd*, which can be a :class:`str` or a + :class:`bytes` string encoded to the + :ref:`filesystem encoding `, + using the platform's "shell" syntax. + + This is similar to the standard library :class:`subprocess.Popen` + class called with ``shell=True``. + + The *protocol_factory* must be a callable returning a subclass of the + :class:`SubprocessProtocol` class. + + See :meth:`~loop.subprocess_exec` for more details about + the remaining arguments. + + Returns a pair of ``(transport, protocol)``, where *transport* + conforms to the :class:`SubprocessTransport` base class and + *protocol* is an object instantiated by the *protocol_factory*. + +.. note:: + It is the application's responsibility to ensure that all whitespace + and special characters are quoted appropriately to avoid `shell injection + `_ + vulnerabilities. The :func:`shlex.quote` function can be used to + properly escape whitespace and special characters in strings that + are going to be used to construct shell commands. + + +Callback Handles +================ + +.. class:: Handle + + A callback wrapper object returned by :meth:`loop.call_soon`, + :meth:`loop.call_soon_threadsafe`. + + .. method:: cancel() + + Cancel the callback. If the callback has already been canceled + or executed, this method has no effect. + + .. method:: cancelled() + + Return ``True`` if the callback was cancelled. + + .. versionadded:: 3.7 + +.. class:: TimerHandle + + A callback wrapper object returned by :meth:`loop.call_later`, + and :meth:`loop.call_at`. + + This class is a subclass of :class:`Handle`. + + .. method:: when() + + Return a scheduled callback time as :class:`float` seconds. + + The time is an absolute timestamp, using the same time + reference as :meth:`loop.time`. + + .. versionadded:: 3.7 + + +Server Objects +============== - Server listening on sockets. +Server objects are created by :meth:`loop.create_server`, +:meth:`loop.create_unix_server`, :func:`start_server`, +and :func:`start_unix_server` functions. - Object created by :meth:`AbstractEventLoop.create_server`, - :meth:`AbstractEventLoop.create_unix_server`, :func:`start_server`, - and :func:`start_unix_server` functions. Don't instantiate the class - directly. +Do not instantiate the class directly. + +.. class:: Server *Server* objects are asynchronous context managers. When used in an ``async with`` statement, it's guaranteed that the Server object is @@ -1022,15 +1314,15 @@ Stop serving: close listening sockets and set the :attr:`sockets` attribute to ``None``. - The sockets that represent existing incoming client connections are left - open. + The sockets that represent existing incoming client connections + are left open. The server is closed asynchronously, use the :meth:`wait_closed` coroutine to wait until the server is closed. .. method:: get_loop() - Gives the event loop associated with the server object. + Return the event loop associated with the server object. .. versionadded:: 3.7 @@ -1041,12 +1333,12 @@ This method is idempotent, so it can be called when the server is already being serving. - The new *start_serving* keyword-only parameter to - :meth:`AbstractEventLoop.create_server` and - :meth:`asyncio.start_server` allows to create a Server object - that is not accepting connections right away. In which case - this method, or :meth:`Server.serve_forever` can be used - to make the Server object to start accepting connections. + The *start_serving* keyword-only parameter to + :meth:`loop.create_server` and + :meth:`asyncio.start_server` allows creating a Server object + that is not accepting connections initially. In this case + ``Server.start_serving()``, or :meth:`Server.serve_forever` can be used + to make the Server start accepting connections. .. versionadded:: 3.7 @@ -1088,78 +1380,99 @@ .. attribute:: sockets - List of :class:`socket.socket` objects the server is listening to, or - ``None`` if the server is closed. + List of :class:`socket.socket` objects the server is listening on, + or ``None`` if the server is closed. .. versionchanged:: 3.7 - Prior to Python 3.7 ``Server.sockets`` used to return the - internal list of server's sockets directly. In 3.7 a copy + Prior to Python 3.7 ``Server.sockets`` used to return an + internal list of server sockets directly. In 3.7 a copy of that list is returned. -Handle ------- +.. _asyncio-event-loops: -.. class:: Handle +Event Loop Implementations +========================== - A callback wrapper object returned by :func:`AbstractEventLoop.call_soon`, - :func:`AbstractEventLoop.call_soon_threadsafe`. +asyncio ships with two different event loop implementations: +:class:`SelectorEventLoop` and :class:`ProactorEventLoop`. - .. method:: cancel() +By default asyncio is configured to use :class:`SelectorEventLoop` +on all platforms. - Cancel the call. If the callback is already canceled or executed, - this method has no effect. - .. method:: cancelled() +.. class:: SelectorEventLoop - Return ``True`` if the call was cancelled. + An event loop based on the :mod:`selectors` module. - .. versionadded:: 3.7 + Uses the most efficient *selector* available for the given + platform. It is also possible to manually configure the + exact selector implementation to be used:: -.. class:: TimerHandle + import asyncio + import selectors - A callback wrapper object returned by :func:`AbstractEventLoop.call_later`, - and :func:`AbstractEventLoop.call_at`. + selector = selectors.SelectSelector() + loop = asyncio.SelectorEventLoop(selector) + asyncio.set_event_loop(loop) - The class is inherited from :class:`Handle`. - .. method:: when() + Availability: Unix, Windows. - Return a scheduled callback time as :class:`float` seconds. - The time is an absolute timestamp, using the same time - reference as :meth:`AbstractEventLoop.time`. +.. class:: ProactorEventLoop - .. versionadded:: 3.7 + An event loop for Windows that uses "I/O Completion Ports" (IOCP). + + Availability: Windows. + + An example how to use :class:`ProactorEventLoop` on Windows:: + + import asyncio + import sys + + if sys.platform == 'win32': + loop = asyncio.ProactorEventLoop() + asyncio.set_event_loop(loop) + + .. seealso:: + + `MSDN documentation on I/O Completion Ports + `_. -SendfileNotAvailableError -------------------------- +.. class:: AbstractEventLoop + Abstract base class for asyncio-compliant event loops. -.. exception:: SendfileNotAvailableError + The :ref:`Event Loop Methods ` section lists all + methods that an alternative implementation of ``AbstractEventLoop`` + should have defined. - Sendfile syscall is not available, subclass of :exc:`RuntimeError`. - Raised if the OS does not support sendfile syscall for - given socket or file type. +Examples +======== +Note that all examples in this section **purposefully** show how +to use the low-level event loop APIs, such as :meth:`loop.run_forever` +and :meth:`loop.call_soon`. Modern asyncio applications rarely +need to be written this way; consider using the high-level functions +like :func:`asyncio.run`. -Event loop examples -------------------- -.. _asyncio-hello-world-callback: +.. _asyncio_example_lowlevel_helloworld: Hello World with call_soon() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Example using the :meth:`AbstractEventLoop.call_soon` method to schedule a -callback. The callback displays ``"Hello World"`` and then stops the event -loop:: +An example using the :meth:`loop.call_soon` method to schedule a +callback. The callback displays ``"Hello World"`` and then stops the +event loop:: import asyncio def hello_world(loop): + """A callback to print 'Hello World' and stop the event loop""" print('Hello World') loop.stop() @@ -1169,23 +1482,25 @@ loop.call_soon(hello_world, loop) # Blocking call interrupted by loop.stop() - loop.run_forever() - loop.close() + try: + loop.run_forever() + finally: + loop.close() .. seealso:: - The :ref:`Hello World coroutine ` example - uses a :ref:`coroutine `. + A similar :ref:`Hello World ` + example created with a coroutine and the :func:`run` function. -.. _asyncio-date-callback: +.. _asyncio_example_call_later: Display the current date with call_later() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Example of callback displaying the current date every second. The callback uses -the :meth:`AbstractEventLoop.call_later` method to reschedule itself during 5 -seconds, and then stops the event loop:: +An example of a callback displaying the current date every second. The +callback uses the :meth:`loop.call_later` method to reschedule itself +after 5 seconds, and then stops the event loop:: import asyncio import datetime @@ -1204,36 +1519,40 @@ loop.call_soon(display_date, end_time, loop) # Blocking call interrupted by loop.stop() - loop.run_forever() - loop.close() + try: + loop.run_forever() + finally: + loop.close() .. seealso:: - The :ref:`coroutine displaying the current date - ` example uses a :ref:`coroutine - `. + A similar :ref:`current date ` example + created with a coroutine and the :func:`run` function. -.. _asyncio-watch-read-event: +.. _asyncio_example_watch_fd: Watch a file descriptor for read events ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Wait until a file descriptor received some data using the -:meth:`AbstractEventLoop.add_reader` method and then close the event loop:: +:meth:`loop.add_reader` method and then close the event loop:: import asyncio from socket import socketpair # Create a pair of connected file descriptors rsock, wsock = socketpair() + loop = asyncio.get_event_loop() def reader(): data = rsock.recv(100) print("Received:", data.decode()) + # We are done: unregister the file descriptor loop.remove_reader(rsock) + # Stop the event loop loop.stop() @@ -1243,30 +1562,35 @@ # Simulate the reception of data from the network loop.call_soon(wsock.send, 'abc'.encode()) - # Run the event loop - loop.run_forever() - - # We are done, close sockets and the event loop - rsock.close() - wsock.close() - loop.close() + try: + # Run the event loop + loop.run_forever() + finally: + # We are done. Close sockets and the event loop. + rsock.close() + wsock.close() + loop.close() .. seealso:: - The :ref:`register an open socket to wait for data using a protocol - ` example uses a low-level protocol created by the - :meth:`AbstractEventLoop.create_connection` method. - - The :ref:`register an open socket to wait for data using streams - ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. + * A similar :ref:`example ` + using transports, protocols, and the + :meth:`loop.create_connection` method. + + * Another similar :ref:`example ` + using the high-level :func:`asyncio.open_connection` function + and streams. +.. _asyncio_example_unix_signals: + Set signal handlers for SIGINT and SIGTERM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM` using -the :meth:`AbstractEventLoop.add_signal_handler` method:: +(This ``signals`` example only works on Unix.) + +Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM` +using the :meth:`loop.add_signal_handler` method:: import asyncio import functools @@ -1277,16 +1601,17 @@ print("got signal %s: exit" % signame) loop.stop() - loop = asyncio.get_event_loop() - for signame in ('SIGINT', 'SIGTERM'): - loop.add_signal_handler(getattr(signal, signame), - functools.partial(ask_exit, signame)) + async def main(): + loop = asyncio.get_running_loop() - print("Event loop running forever, press Ctrl+C to interrupt.") - print("pid %s: send SIGINT or SIGTERM to exit." % os.getpid()) - try: - loop.run_forever() - finally: - loop.close() + for signame in {'SIGINT', 'SIGTERM'}: + loop.add_signal_handler( + getattr(signal, signame), + functools.partial(ask_exit, signame)) + + await asyncio.sleep(3600) + + print("Event loop running for 1 hour, press Ctrl+C to interrupt.") + print(f"pid {os.getpid()}: send SIGINT or SIGTERM to exit.") -This example only works on UNIX. + asyncio.run(main()) diff -Nru python3.7-3.7.0/Doc/library/asyncio-eventloops.rst python3.7-3.7.1/Doc/library/asyncio-eventloops.rst --- python3.7-3.7.0/Doc/library/asyncio-eventloops.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-eventloops.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,244 +0,0 @@ -.. currentmodule:: asyncio - -Event loops -=========== - -**Source code:** :source:`Lib/asyncio/events.py` - -Event loop functions --------------------- - -The following functions are convenient shortcuts to accessing the methods of the -global policy. Note that this provides access to the default policy, unless an -alternative policy was set by calling :func:`set_event_loop_policy` earlier in -the execution of the process. - -.. function:: get_event_loop() - - Equivalent to calling ``get_event_loop_policy().get_event_loop()``. - -.. function:: set_event_loop(loop) - - Equivalent to calling ``get_event_loop_policy().set_event_loop(loop)``. - -.. function:: new_event_loop() - - Equivalent to calling ``get_event_loop_policy().new_event_loop()``. - -.. function:: get_running_loop() - - Return the running event loop in the current OS thread. If there - is no running event loop a :exc:`RuntimeError` is raised. - - .. versionadded:: 3.7 - - -.. _asyncio-event-loops: - -Available event loops ---------------------- - -asyncio currently provides two implementations of event loops: -:class:`SelectorEventLoop` and :class:`ProactorEventLoop`. - -.. class:: SelectorEventLoop - - Event loop based on the :mod:`selectors` module. Subclass of - :class:`AbstractEventLoop`. - - Use the most efficient selector available on the platform. - - On Windows, only sockets are supported (ex: pipes are not supported): - see the `MSDN documentation of select - `_. - -.. class:: ProactorEventLoop - - Proactor event loop for Windows using "I/O Completion Ports" aka IOCP. - Subclass of :class:`AbstractEventLoop`. - - Availability: Windows. - - .. seealso:: - - `MSDN documentation on I/O Completion Ports - `_. - -Example to use a :class:`ProactorEventLoop` on Windows:: - - import asyncio, sys - - if sys.platform == 'win32': - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - -.. _asyncio-platform-support: - -Platform support ----------------- - -The :mod:`asyncio` module has been designed to be portable, but each platform -still has subtle differences and may not support all :mod:`asyncio` features. - -Windows -^^^^^^^ - -Common limits of Windows event loops: - -- :meth:`~AbstractEventLoop.create_unix_connection` and - :meth:`~AbstractEventLoop.create_unix_server` are not supported: the socket - family :data:`socket.AF_UNIX` is specific to UNIX -- :meth:`~AbstractEventLoop.add_signal_handler` and - :meth:`~AbstractEventLoop.remove_signal_handler` are not supported -- :meth:`EventLoopPolicy.set_child_watcher` is not supported. - :class:`ProactorEventLoop` supports subprocesses. It has only one - implementation to watch child processes, there is no need to configure it. - -:class:`SelectorEventLoop` specific limits: - -- :class:`~selectors.SelectSelector` is used which only supports sockets - and is limited to 512 sockets. -- :meth:`~AbstractEventLoop.add_reader` and :meth:`~AbstractEventLoop.add_writer` only - accept file descriptors of sockets -- Pipes are not supported - (ex: :meth:`~AbstractEventLoop.connect_read_pipe`, - :meth:`~AbstractEventLoop.connect_write_pipe`) -- :ref:`Subprocesses ` are not supported - (ex: :meth:`~AbstractEventLoop.subprocess_exec`, - :meth:`~AbstractEventLoop.subprocess_shell`) - -:class:`ProactorEventLoop` specific limits: - -- :meth:`~AbstractEventLoop.create_datagram_endpoint` (UDP) is not supported -- :meth:`~AbstractEventLoop.add_reader` and :meth:`~AbstractEventLoop.add_writer` are - not supported - -The resolution of the monotonic clock on Windows is usually around 15.6 msec. -The best resolution is 0.5 msec. The resolution depends on the hardware -(availability of `HPET -`_) and on the Windows -configuration. See :ref:`asyncio delayed calls `. - -.. versionchanged:: 3.5 - - :class:`ProactorEventLoop` now supports SSL. - - -Mac OS X -^^^^^^^^ - -Character devices like PTY are only well supported since Mavericks (Mac OS -10.9). They are not supported at all on Mac OS 10.5 and older. - -On Mac OS 10.6, 10.7 and 10.8, the default event loop is -:class:`SelectorEventLoop` which uses :class:`selectors.KqueueSelector`. -:class:`selectors.KqueueSelector` does not support character devices on these -versions. The :class:`SelectorEventLoop` can be used with -:class:`~selectors.SelectSelector` or :class:`~selectors.PollSelector` to -support character devices on these versions of Mac OS X. Example:: - - import asyncio - import selectors - - selector = selectors.SelectSelector() - loop = asyncio.SelectorEventLoop(selector) - asyncio.set_event_loop(loop) - - -Event loop policies and the default policy ------------------------------------------- - -Event loop management is abstracted with a *policy* pattern, to provide maximal -flexibility for custom platforms and frameworks. Throughout the execution of a -process, a single global policy object manages the event loops available to the -process based on the calling context. A policy is an object implementing the -:class:`AbstractEventLoopPolicy` interface. - -For most users of :mod:`asyncio`, policies never have to be dealt with -explicitly, since the default global policy is sufficient (see below). - -The module-level functions -:func:`get_event_loop` and :func:`set_event_loop` provide convenient access to -event loops managed by the default policy. - - -Event loop policy interface ---------------------------- - -An event loop policy must implement the following interface: - -.. class:: AbstractEventLoopPolicy - - Event loop policy. - - .. method:: get_event_loop() - - Get the event loop for the current context. - - Returns an event loop object implementing the :class:`AbstractEventLoop` - interface. In case called from coroutine, it returns the currently - running event loop. - - Raises an exception in case no event loop has been set for the current - context and the current policy does not specify to create one. It must - never return ``None``. - - .. versionchanged:: 3.6 - - .. method:: set_event_loop(loop) - - Set the event loop for the current context to *loop*. - - .. method:: new_event_loop() - - Create and return a new event loop object according to this policy's - rules. - - If there's need to set this loop as the event loop for the current - context, :meth:`set_event_loop` must be called explicitly. - - -The default policy defines context as the current thread, and manages an event -loop per thread that interacts with :mod:`asyncio`. An exception to this rule -happens when :meth:`~AbstractEventLoopPolicy.get_event_loop` is called from a -running future/coroutine, in which case it will return the current loop -running that future/coroutine. - -If the current thread doesn't already have an event loop associated with it, -the default policy's :meth:`~AbstractEventLoopPolicy.get_event_loop` method -creates one when called from the main thread, but raises :exc:`RuntimeError` -otherwise. - - -Access to the global loop policy --------------------------------- - -.. function:: get_event_loop_policy() - - Get the current event loop policy. - -.. function:: set_event_loop_policy(policy) - - Set the current event loop policy. If *policy* is ``None``, the default - policy is restored. - - -Customizing the event loop policy ---------------------------------- - -To implement a new event loop policy, it is recommended you subclass the -concrete default event loop policy :class:`DefaultEventLoopPolicy` -and override the methods for which you want to change behavior, for example:: - - class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy): - - def get_event_loop(self): - """Get the event loop. - - This may be None or an instance of EventLoop. - """ - loop = super().get_event_loop() - # Do something with loop ... - return loop - - asyncio.set_event_loop_policy(MyEventLoopPolicy()) diff -Nru python3.7-3.7.0/Doc/library/asyncio-exceptions.rst python3.7-3.7.1/Doc/library/asyncio-exceptions.rst --- python3.7-3.7.0/Doc/library/asyncio-exceptions.rst 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-exceptions.rst 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,91 @@ +.. currentmodule:: asyncio + + +.. _asyncio-exceptions: + +========== +Exceptions +========== + + +.. exception:: TimeoutError + + The operation has exceeded the given deadline. + + .. important:: + This exception is different from the builtin :exc:`TimeoutError` + exception. + + +.. exception:: CancelledError + + The operation has been cancelled. + + This exception can be caught to perform custom operations + when asyncio Tasks are cancelled. In almost all situations the + exception must be re-raised. + + .. important:: + + This exception is a subclass of :exc:`Exception`, so it can be + accidentally suppressed by an overly broad ``try..except`` block:: + + try: + await operation + except Exception: + # The cancellation is broken because the *except* block + # suppresses the CancelledError exception. + log.log('an error has occurred') + + Instead, the following pattern should be used:: + + try: + await operation + except asyncio.CancelledError: + raise + except Exception: + log.log('an error has occurred') + + +.. exception:: InvalidStateError + + Invalid internal state of :class:`Task` or :class:`Future`. + + Can be raised in situations like setting a result value for a + *Future* object that already has a result value set. + + +.. exception:: SendfileNotAvailableError + + The "sendfile" syscall is not available for the given + socket or file type. + + A subclass of :exc:`RuntimeError`. + + +.. exception:: IncompleteReadError + + The requested read operation did not complete fully. + + Raised by the :ref:`asyncio stream APIs`. + + This exception is a subclass of :exc:`EOFError`. + + .. attribute:: expected + + The total number (:class:`int`) of expected bytes. + + .. attribute:: partial + + A string of :class:`bytes` read before the end of stream was reached. + + +.. exception:: LimitOverrunError + + Reached the buffer size limit while looking for a separator. + + Raised by the :ref:`asyncio stream APIs `. + + .. attribute:: consumed + + The total number of to be consumed bytes. diff -Nru python3.7-3.7.0/Doc/library/asyncio-future.rst python3.7-3.7.1/Doc/library/asyncio-future.rst --- python3.7-3.7.0/Doc/library/asyncio-future.rst 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-future.rst 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,250 @@ +.. currentmodule:: asyncio + + +.. _asyncio-futures: + +======= +Futures +======= + +*Future* objects are used to bridge **low-level callback-based code** +with high-level async/await code. + + +Future Functions +================ + +.. function:: isfuture(obj) + + Return ``True`` if *obj* is either of: + + * an instance of :class:`asyncio.Future`, + * an instance of :class:`asyncio.Task`, + * a Future-like object with a ``_asyncio_future_blocking`` + attribute. + + .. versionadded:: 3.5 + + +.. function:: ensure_future(obj, \*, loop=None) + + Return: + + * *obj* argument as is, if *obj* is a :class:`Future`, + a :class:`Task`, or a Future-like object (:func:`isfuture` + is used for the test.) + + * a :class:`Task` object wrapping *obj*, if *obj* is a + coroutine (:func:`iscoroutine` is used for the test.) + + * a :class:`Task` object that would await on *obj*, if *obj* is an + awaitable (:func:`inspect.isawaitable` is used for the test.) + + If *obj* is neither of the above a :exc:`TypeError` is raised. + + .. important:: + + See also the :func:`create_task` function which is the + preferred way for creating new Tasks. + + .. versionchanged:: 3.5.1 + The function accepts any :term:`awaitable` object. + + +.. function:: wrap_future(future, \*, loop=None) + + Wrap a :class:`concurrent.futures.Future` object in a + :class:`asyncio.Future` object. + + +Future Object +============= + +.. class:: Future(\*, loop=None) + + A Future represents an eventual result of an asynchronous + operation. Not thread-safe. + + Future is an :term:`awaitable` object. Coroutines can await on + Future objects until they either have a result or an exception + set, or until they are cancelled. + + Typically Futures are used to enable low-level + callback-based code (e.g. in protocols implemented using asyncio + :ref:`transports `) + to interoperate with high-level async/await code. + + The rule of thumb is to never expose Future objects in user-facing + APIs, and the recommended way to create a Future object is to call + :meth:`loop.create_future`. This way alternative event loop + implementations can inject their own optimized implementations + of a Future object. + + .. versionchanged:: 3.7 + Added support for the :mod:`contextvars` module. + + .. method:: result() + + Return the result of the Future. + + If the Future is *done* and has a result set by the + :meth:`set_result` method, the result value is returned. + + If the Future is *done* and has an exception set by the + :meth:`set_exception` method, this method raises the exception. + + If the Future has been *cancelled*, this method raises + a :exc:`CancelledError` exception. + + If the Future's result isn't yet available, this method raises + a :exc:`InvalidStateError` exception. + + .. method:: set_result(result) + + Mark the Future as *done* and set its result. + + Raises a :exc:`InvalidStateError` error if the Future is + already *done*. + + .. method:: set_exception(exception) + + Mark the Future as *done* and set an exception. + + Raises a :exc:`InvalidStateError` error if the Future is + already *done*. + + .. method:: done() + + Return ``True`` if the Future is *done*. + + A Future is *done* if it was *cancelled* or if it has a result + or an exception set with :meth:`set_result` or + :meth:`set_exception` calls. + + .. method:: cancelled() + + Return ``True`` if the Future was *cancelled*. + + The method is usually used to check if a Future is not + *cancelled* before setting a result or an exception for it:: + + if not fut.cancelled(): + fut.set_result(42) + + .. method:: add_done_callback(callback, *, context=None) + + Add a callback to be run when the Future is *done*. + + The *callback* is called with the Future object as its only + argument. + + If the Future is already *done* when this method is called, + the callback is scheduled with :meth:`loop.call_soon`. + + An optional keyword-only *context* argument allows specifying a + custom :class:`contextvars.Context` for the *callback* to run in. + The current context is used when no *context* is provided. + + :func:`functools.partial` can be used to pass parameters + to the callback, e.g.:: + + # Call 'print("Future:", fut)' when "fut" is done. + fut.add_done_callback( + functools.partial(print, "Future:")) + + .. versionchanged:: 3.7 + The *context* keyword-only parameter was added. + See :pep:`567` for more details. + + .. method:: remove_done_callback(callback) + + Remove *callback* from the callbacks list. + + Returns the number of callbacks removed, which is typically 1, + unless a callback was added more than once. + + .. method:: cancel() + + Cancel the Future and schedule callbacks. + + If the Future is already *done* or *cancelled*, return ``False``. + Otherwise, change the Future's state to *cancelled*, + schedule the callbacks, and return ``True``. + + .. method:: exception() + + Return the exception that was set on this Future. + + The exception (or ``None`` if no exception was set) is + returned only if the Future is *done*. + + If the Future has been *cancelled*, this method raises a + :exc:`CancelledError` exception. + + If the Future isn't *done* yet, this method raises an + :exc:`InvalidStateError` exception. + + .. method:: get_loop() + + Return the event loop the Future object is bound to. + + .. versionadded:: 3.7 + + +.. _asyncio_example_future: + +This example creates a Future object, creates and schedules an +asynchronous Task to set result for the Future, and waits until +the Future has a result:: + + async def set_after(fut, delay, value): + # Sleep for *delay* seconds. + await asyncio.sleep(delay) + + # Set *value* as a result of *fut* Future. + fut.set_result(value) + + async def main(): + # Get the current event loop. + loop = asyncio.get_running_loop() + + # Create a new Future object. + fut = loop.create_future() + + # Run "set_after()" coroutine in a parallel Task. + # We are using the low-level "loop.create_task()" API here because + # we already have a reference to the event loop at hand. + # Otherwise we could have just used "asyncio.create_task()". + loop.create_task( + set_after(fut, 1, '... world')) + + print('hello ...') + + # Wait until *fut* has a result (1 second) and print it. + print(await fut) + + asyncio.run(main()) + + +.. important:: + + The Future object was designed to mimic + :class:`concurrent.futures.Future`. Key differences include: + + - unlike asyncio Futures, :class:`concurrent.futures.Future` + instances cannot be awaited. + + - :meth:`asyncio.Future.result` and :meth:`asyncio.Future.exception` + do not accept the *timeout* argument. + + - :meth:`asyncio.Future.result` and :meth:`asyncio.Future.exception` + raise an :exc:`InvalidStateError` exception when the Future is not + *done*. + + - Callbacks registered with :meth:`asyncio.Future.add_done_callback` + are not called immediately. They are scheduled with + :meth:`loop.call_soon` instead. + + - asyncio Future is not compatible with the + :func:`concurrent.futures.wait` and + :func:`concurrent.futures.as_completed` functions. diff -Nru python3.7-3.7.0/Doc/library/asyncio-llapi-index.rst python3.7-3.7.1/Doc/library/asyncio-llapi-index.rst --- python3.7-3.7.0/Doc/library/asyncio-llapi-index.rst 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-llapi-index.rst 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,510 @@ +.. currentmodule:: asyncio + + +=================== +Low-level API Index +=================== + +This page lists all low-level asyncio APIs. + + +Obtaining the Event Loop +======================== + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :func:`asyncio.get_running_loop` + - The **preferred** function to get the running event loop. + + * - :func:`asyncio.get_event_loop` + - Get an event loop instance (current or via the policy). + + * - :func:`asyncio.set_event_loop` + - Set the event loop as current via the current policy. + + * - :func:`asyncio.new_event_loop` + - Create a new event loop. + + +.. rubric:: Examples + +* :ref:`Using asyncio.get_running_loop() `. + + +Event Loop Methods +================== + +See also the main documentation section about the +:ref:`event loop methods `. + +.. rubric:: Lifecycle +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`loop.run_until_complete` + - Run a Future/Task/awaitable until complete. + + * - :meth:`loop.run_forever` + - Run the event loop forever. + + * - :meth:`loop.stop` + - Stop the event loop. + + * - :meth:`loop.close` + - Close the event loop. + + * - :meth:`loop.is_running()` + - Return ``True`` if the event loop is running. + + * - :meth:`loop.is_closed()` + - Return ``True`` if the event loop is closed. + + * - ``await`` :meth:`loop.shutdown_asyncgens` + - Close asynchronous generators. + + +.. rubric:: Debugging +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`loop.set_debug` + - Enable or disable the debug mode. + + * - :meth:`loop.get_debug` + - Get the current debug mode. + + +.. rubric:: Scheduling Callbacks +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`loop.call_soon` + - Invoke a callback soon. + + * - :meth:`loop.call_soon_threadsafe` + - A thread-safe variant of :meth:`loop.call_soon`. + + * - :meth:`loop.call_later` + - Invoke a callback *after* the given time. + + * - :meth:`loop.call_at` + - Invoke a callback *at* the given time. + + +.. rubric:: Thread/Process Pool +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :meth:`loop.run_in_executor` + - Run a CPU-bound or other blocking function in + a :mod:`concurrent.futures` executor. + + * - :meth:`loop.set_default_executor` + - Set the default executor for :meth:`loop.run_in_executor`. + + +.. rubric:: Tasks and Futures +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`loop.create_future` + - Create a :class:`Future` object. + + * - :meth:`loop.create_task` + - Schedule coroutine as a :class:`Task`. + + * - :meth:`loop.set_task_factory` + - Set a factory used by :meth:`loop.create_task` to + create :class:`Tasks `. + + * - :meth:`loop.get_task_factory` + - Get the factory :meth:`loop.create_task` uses + to create :class:`Tasks `. + + +.. rubric:: DNS +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :meth:`loop.getaddrinfo` + - Asynchronous version of :meth:`socket.getaddrinfo`. + + * - ``await`` :meth:`loop.getnameinfo` + - Asynchronous version of :meth:`socket.getnameinfo`. + + +.. rubric:: Networking and IPC +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :meth:`loop.create_connection` + - Open a TCP connection. + + * - ``await`` :meth:`loop.create_server` + - Create a TCP server. + + * - ``await`` :meth:`loop.create_unix_connection` + - Open a Unix socket connection. + + * - ``await`` :meth:`loop.create_unix_server` + - Create a Unix socket server. + + * - ``await`` :meth:`loop.connect_accepted_socket` + - Wrap a :class:`~socket.socket` into a ``(transport, protocol)`` + pair. + + * - ``await`` :meth:`loop.create_datagram_endpoint` + - Open a datagram (UDP) connection. + + * - ``await`` :meth:`loop.sendfile` + - Send a file over a transport. + + * - ``await`` :meth:`loop.start_tls` + - Upgrade an existing connection to TLS. + + * - ``await`` :meth:`loop.connect_read_pipe` + - Wrap a read end of a pipe into a ``(transport, protocol)`` pair. + + * - ``await`` :meth:`loop.connect_write_pipe` + - Wrap a write end of a pipe into a ``(transport, protocol)`` pair. + + +.. rubric:: Sockets +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :meth:`loop.sock_recv` + - Receive data from the :class:`~socket.socket`. + + * - ``await`` :meth:`loop.sock_recv_into` + - Receive data from the :class:`~socket.socket` into a buffer. + + * - ``await`` :meth:`loop.sock_sendall` + - Send data to the :class:`~socket.socket`. + + * - ``await`` :meth:`loop.sock_connect` + - Connect the :class:`~socket.socket`. + + * - ``await`` :meth:`loop.sock_accept` + - Accept a :class:`~socket.socket` connection. + + * - ``await`` :meth:`loop.sock_sendfile` + - Send a file over the :class:`~socket.socket`. + + * - :meth:`loop.add_reader` + - Start watching a file descriptor for read availability. + + * - :meth:`loop.remove_reader` + - Stop watching a file descriptor for read availability. + + * - :meth:`loop.add_writer` + - Start watching a file descriptor for write availability. + + * - :meth:`loop.remove_writer` + - Stop watching a file descriptor for write availability. + + +.. rubric:: Unix Signals +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`loop.add_signal_handler` + - Add a handler for a :mod:`signal`. + + * - :meth:`loop.remove_signal_handler` + - Remove a handler for a :mod:`signal`. + + +.. rubric:: Subprocesses +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`loop.subprocess_exec` + - Spawn a subprocess. + + * - :meth:`loop.subprocess_shell` + - Spawn a subprocess from a shell command. + + +.. rubric:: Error Handling +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`loop.call_exception_handler` + - Call the exception handler. + + * - :meth:`loop.set_exception_handler` + - Set a new exception handler. + + * - :meth:`loop.get_exception_handler` + - Get the current exception handler. + + * - :meth:`loop.default_exception_handler` + - The default exception handler implementation. + + +.. rubric:: Examples + +* :ref:`Using asyncio.get_event_loop() and loop.run_forever() + `. + +* :ref:`Using loop.call_later() `. + +* Using ``loop.create_connection()`` to implement + :ref:`an echo-client `. + +* Using ``loop.create_connection()`` to + :ref:`connect a socket `. + +* :ref:`Using add_reader() to watch an FD for read events + `. + +* :ref:`Using loop.add_signal_handler() `. + +* :ref:`Using loop.subprocess_exec() `. + + +Transports +========== + +All transports implement the following methods: + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`transport.close() ` + - Close the transport. + + * - :meth:`transport.is_closing() ` + - Return ``True`` if the transport is closing or is closed. + + * - :meth:`transport.get_extra_info() ` + - Request for information about the transport. + + * - :meth:`transport.set_protocol() ` + - Set a new protocol. + + * - :meth:`transport.get_protocol() ` + - Return the current protocol. + + +Transports that can receive data (TCP and Unix connections, +pipes, etc). Returned from methods like +:meth:`loop.create_connection`, :meth:`loop.create_unix_connection`, +:meth:`loop.connect_read_pipe`, etc: + +.. rubric:: Read Transports +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`transport.is_reading() ` + - Return ``True`` if the transport is receiving. + + * - :meth:`transport.pause_reading() ` + - Pause receiving. + + * - :meth:`transport.resume_reading() ` + - Resume receiving. + + +Transports that can Send data (TCP and Unix connections, +pipes, etc). Returned from methods like +:meth:`loop.create_connection`, :meth:`loop.create_unix_connection`, +:meth:`loop.connect_write_pipe`, etc: + +.. rubric:: Write Transports +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`transport.write() ` + - Write data to the transport. + + * - :meth:`transport.writelines() ` + - Write buffers to the transport. + + * - :meth:`transport.can_write_eof() ` + - Return :const:`True` if the transport supports sending EOF. + + * - :meth:`transport.write_eof() ` + - Close and send EOF after flushing buffered data. + + * - :meth:`transport.abort() ` + - Close the transport immediately. + + * - :meth:`transport.get_write_buffer_size() + ` + - Return high and low water marks for write flow control. + + * - :meth:`transport.set_write_buffer_limits() + ` + - Set new high and low water marks for write flow control. + + +Transports returned by :meth:`loop.create_datagram_endpoint`: + +.. rubric:: Datagram Transports +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`transport.sendto() ` + - Send data to the remote peer. + + * - :meth:`transport.abort() ` + - Close the transport immediately. + + +Low-level transport abstraction over subprocesses. +Returned by :meth:`loop.subprocess_exec` and +:meth:`loop.subprocess_shell`: + +.. rubric:: Subprocess Transports +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`transport.get_pid() ` + - Return the subprocess process id. + + * - :meth:`transport.get_pipe_transport() + ` + - Return the transport for the requested communication pipe + (*stdin*, *stdout*, or *stderr*). + + * - :meth:`transport.get_returncode() ` + - Return the subprocess return code. + + * - :meth:`transport.kill() ` + - Kill the subprocess. + + * - :meth:`transport.send_signal() ` + - Send a signal to the subprocess. + + * - :meth:`transport.terminate() ` + - Stop the subprocess. + + * - :meth:`transport.close() ` + - Kill the subprocess and close all pipes. + + +Protocols +========= + +Protocol classes can implement the following **callback methods**: + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``callback`` :meth:`connection_made() ` + - Called when a connection is made. + + * - ``callback`` :meth:`connection_lost() ` + - Called when the connection is lost or closed. + + * - ``callback`` :meth:`pause_writing() ` + - Called when the transport's buffer goes over the high water mark. + + * - ``callback`` :meth:`resume_writing() ` + - Called when the transport's buffer drains below the low water mark. + + +.. rubric:: Streaming Protocols (TCP, Unix Sockets, Pipes) +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``callback`` :meth:`data_received() ` + - Called when some data is received. + + * - ``callback`` :meth:`eof_received() ` + - Called when an EOF is received. + + +.. rubric:: Buffered Streaming Protocols +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``callback`` :meth:`get_buffer() ` + - Called to allocate a new receive buffer. + + * - ``callback`` :meth:`buffer_updated() ` + - Called when the buffer was updated with the received data. + + * - ``callback`` :meth:`eof_received() ` + - Called when an EOF is received. + + +.. rubric:: Datagram Protocols +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``callback`` :meth:`datagram_received() + ` + - Called when a datagram is received. + + * - ``callback`` :meth:`error_received() ` + - Called when a previous send or receive operation raises an + :class:`OSError`. + + +.. rubric:: Subprocess Protocols +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``callback`` :meth:`pipe_data_received() + ` + - Called when the child process writes data into its + *stdout* or *stderr* pipe. + + * - ``callback`` :meth:`pipe_connection_lost() + ` + - Called when one of the pipes communicating with + the child process is closed. + + * - ``callback`` :meth:`process_exited() + ` + - Called when the child process has exited. + + +Event Loop Policies +=================== + +Policies is a low-level mechanism to alter the behavior of +functions like :func:`asyncio.get_event_loop`. See also +the main :ref:`policies section ` for more +details. + + +.. rubric:: Accessing Policies +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :meth:`asyncio.get_event_loop_policy` + - Return the current process-wide policy. + + * - :meth:`asyncio.set_event_loop_policy` + - Set a new process-wide policy. + + * - :class:`AbstractEventLoopPolicy` + - Base class for policy objects. diff -Nru python3.7-3.7.0/Doc/library/asyncio-platforms.rst python3.7-3.7.1/Doc/library/asyncio-platforms.rst --- python3.7-3.7.0/Doc/library/asyncio-platforms.rst 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-platforms.rst 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,106 @@ +.. currentmodule:: asyncio + + +.. _asyncio-platform-support: + + +================ +Platform Support +================ + +The :mod:`asyncio` module is designed to be portable, +but some platforms have subtle differences and limitations +due to the platforms' underlying architecture and capabilities. + + +All Platforms +============= + +* :meth:`loop.add_reader` and :meth:`loop.add_writer` + cannot be used to monitor file I/O. + + +Windows +======= + +All event loops on Windows do not support the following methods: + +* :meth:`loop.create_unix_connection` and + :meth:`loop.create_unix_server` are not supported. + The :data:`socket.AF_UNIX` socket family is specific to Unix. + +* :meth:`loop.add_signal_handler` and + :meth:`loop.remove_signal_handler` are not supported. + +:class:`SelectorEventLoop` has the following limitations: + +* :class:`~selectors.SelectSelector` is used to wait on socket events: + it supports sockets and is limited to 512 sockets. + +* :meth:`loop.add_reader` and :meth:`loop.add_writer` only accept + socket handles (e.g. pipe file descriptors are not supported). + +* Pipes are not supported, so the :meth:`loop.connect_read_pipe` + and :meth:`loop.connect_write_pipe` methods are not implemented. + +* :ref:`Subprocesses ` are not supported, i.e. + :meth:`loop.subprocess_exec` and :meth:`loop.subprocess_shell` + methods are not implemented. + +:class:`ProactorEventLoop` has the following limitations: + +* The :meth:`loop.create_datagram_endpoint` method + is not supported. + +* The :meth:`loop.add_reader` and :meth:`loop.add_writer` + methods are not supported. + +The resolution of the monotonic clock on Windows is usually around 15.6 +msec. The best resolution is 0.5 msec. The resolution depends on the +hardware (availability of `HPET +`_) and on the +Windows configuration. + + +.. _asyncio-windows-subprocess: + +Subprocess Support on Windows +----------------------------- + +:class:`SelectorEventLoop` on Windows does not support subproceses. +On Windows, :class:`ProactorEventLoop` should be used instead:: + + import asyncio + + asyncio.set_event_loop_policy( + asyncio.WindowsProactorEventLoopPolicy()) + + asyncio.run(your_code()) + + +The :meth:`policy.set_child_watcher() +` function is also +not supported, as :class:`ProactorEventLoop` has a different mechanism +to watch child processes. + + +macOS +===== + +Modern macOS versions are fully supported. + +.. rubric:: macOS <= 10.8 + +On macOS 10.6, 10.7 and 10.8, the default event loop +uses :class:`selectors.KqueueSelector`, which does not support +character devices on these versions. The :class:`SelectorEventLoop` +can be manually configured to use :class:`~selectors.SelectSelector` +or :class:`~selectors.PollSelector` to support character devices on +these older versions of macOS. Example:: + + import asyncio + import selectors + + selector = selectors.SelectSelector() + loop = asyncio.SelectorEventLoop(selector) + asyncio.set_event_loop(loop) diff -Nru python3.7-3.7.0/Doc/library/asyncio-policy.rst python3.7-3.7.1/Doc/library/asyncio-policy.rst --- python3.7-3.7.0/Doc/library/asyncio-policy.rst 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-policy.rst 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,221 @@ +.. currentmodule:: asyncio + + +.. _asyncio-policies: + +======== +Policies +======== + +An event loop policy is a global per-process object that controls +the management of the event loop. Each event loop has a default +policy, which can be changed and customized using the policy API. + +A policy defines the notion of *context* and manages a +separate event loop per context. The default policy +defines *context* to be the current thread. + +By using a custom event loop policy, the behavior of +:func:`get_event_loop`, :func:`set_event_loop`, and +:func:`new_event_loop` functions can be customized. + +Policy objects should implement the APIs defined +in the :class:`AbstractEventLoopPolicy` abstract base class. + + +Getting and Setting the Policy +============================== + +The following functions can be used to get and set the policy +for the current process: + +.. function:: get_event_loop_policy() + + Return the current process-wide policy. + +.. function:: set_event_loop_policy(policy) + + Set the current process-wide policy to *policy*. + + If *policy* is set to ``None``, the default policy is restored. + + +Policy Objects +============== + +The abstract event loop policy base class is defined as follows: + +.. class:: AbstractEventLoopPolicy + + An abstract base class for asyncio policies. + + .. method:: get_event_loop() + + Get the event loop for the current context. + + Return an event loop object implementing the + :class:`AbstractEventLoop` interface. + + This method should never return ``None``. + + .. versionchanged:: 3.6 + + .. method:: set_event_loop(loop) + + Set the event loop for the current context to *loop*. + + .. method:: new_event_loop() + + Create and return a new event loop object. + + This method should never return ``None``. + + .. method:: get_child_watcher() + + Get a child process watcher object. + + Return a watcher object implementing the + :class:`AbstractChildWatcher` interface. + + This function is Unix specific. + + .. method:: set_child_watcher(watcher) + + Get the current child process watcher to *watcher*. + + This function is Unix specific. + + +asyncio ships with the following built-in policies: + + +.. class:: DefaultEventLoopPolicy + + The default asyncio policy. Uses :class:`SelectorEventLoop` + on both Unix and Windows platforms. + + There is no need to install the default policy manually. asyncio + is configured to use the default policy automatically. + + +.. class:: WindowsProactorEventLoopPolicy + + An alternative event loop policy that uses the + :class:`ProactorEventLoop` event loop implementation. + + Availability: Windows. + + +Process Watchers +================ + +A process watcher allows customization of how an event loop monitors +child processes on Unix. Specifically, the event loop needs to know +when a child process has exited. + +In asyncio, child processes are created with +:func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` +functions. + +asyncio defines the :class:`AbstractChildWatcher` abstract base class, +which child watchers should implement, and has two different +implementations: :class:`SafeChildWatcher` (configured to be used +by default) and :class:`FastChildWatcher`. + +See also the :ref:`Subprocess and Threads ` +section. + +The following two functions can be used to customize the child process watcher +implementation used by the asyncio event loop: + +.. function:: get_child_watcher() + + Return the current child watcher for the current policy. + +.. function:: set_child_watcher(watcher) + + Set the current child watcher to *watcher* for the current + policy. *watcher* must implement methods defined in the + :class:`AbstractChildWatcher` base class. + +.. note:: + Third-party event loops implementations might not support + custom child watchers. For such event loops, using + :func:`set_child_watcher` might be prohibited or have no effect. + +.. class:: AbstractChildWatcher + + .. method:: add_child_handler(pid, callback, \*args) + + Register a new child handler. + + Arrange for ``callback(pid, returncode, *args)`` to be called + when a process with PID equal to *pid* terminates. Specifying + another callback for the same process replaces the previous + handler. + + The *callback* callable must be thread-safe. + + .. method:: remove_child_handler(pid) + + Removes the handler for process with PID equal to *pid*. + + The function returns ``True`` if the handler was successfully + removed, ``False`` if there was nothing to remove. + + .. method:: attach_loop(loop) + + Attach the watcher to an event loop. + + If the watcher was previously attached to an event loop, then + it is first detached before attaching to the new loop. + + Note: loop may be ``None``. + + .. method:: close() + + Close the watcher. + + This method has to be called to ensure that underlying + resources are cleaned-up. + +.. class:: SafeChildWatcher + + This implementation avoids disrupting other code spawning processes + by polling every process explicitly on a :py:data:`SIGCHLD` signal. + + This is a safe solution but it has a significant overhead when + handling a big number of processes (*O(n)* each time a + :py:data:`SIGCHLD` is received). + + asyncio uses this safe implementation by default. + +.. class:: FastChildWatcher + + This implementation reaps every terminated processes by calling + ``os.waitpid(-1)`` directly, possibly breaking other code spawning + processes and waiting for their termination. + + There is no noticeable overhead when handling a big number of + children (*O(1)* each time a child terminates). + + +Custom Policies +=============== + +To implement a new event loop policy, it is recommended to subclass +:class:`DefaultEventLoopPolicy` and override the methods for which +custom behavior is wanted, e.g.:: + + class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy): + + def get_event_loop(self): + """Get the event loop. + + This may be None or an instance of EventLoop. + """ + loop = super().get_event_loop() + # Do something with loop ... + return loop + + asyncio.set_event_loop_policy(MyEventLoopPolicy()) diff -Nru python3.7-3.7.0/Doc/library/asyncio-protocol.rst python3.7-3.7.1/Doc/library/asyncio-protocol.rst --- python3.7-3.7.0/Doc/library/asyncio-protocol.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-protocol.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,12 +1,68 @@ .. currentmodule:: asyncio -+++++++++++++++++++++++++++++++++++++++++++++ -Transports and protocols (callback based API) -+++++++++++++++++++++++++++++++++++++++++++++ -**Source code:** :source:`Lib/asyncio/transports.py` +.. _asyncio-transports-protocols: + + +======================== +Transports and Protocols +======================== + +.. rubric:: Preface + +Transports and Protocols are used by the **low-level** event loop +APIs such as :meth:`loop.create_connection`. They use +callback-based programming style and enable high-performance +implementations of network or IPC protocols (e.g. HTTP). + +Essentially, transports and protocols should only be used in +libraries and frameworks and never in high-level asyncio +applications. + +This documentation page covers both `Transports`_ and `Protocols`_. + +.. rubric:: Introduction + +At the highest level, the transport is concerned with *how* bytes +are transmitted, while the protocol determines *which* bytes to +transmit (and to some extent when). + +A different way of saying the same thing: a transport is an +abstraction for a socket (or similar I/O endpoint) while a protocol +is an abstraction for an application, from the transport's point +of view. + +Yet another view is the transport and protocol interfaces +together define an abstract interface for using network I/O and +interprocess I/O. + +There is always a 1:1 relationship between transport and protocol +objects: the protocol calls transport methods to send data, +while the transport calls protocol methods to pass it data that +has been received. + +Most of connection oriented event loop methods +(such as :meth:`loop.create_connection`) usually accept a +*protocol_factory* argument used to create a *Protocol* object +for an accepted connection, represented by a *Transport* object. +Such methods usually return a tuple of ``(transport, protocol)``. + +.. rubric:: Contents + +This documentation page contains the following sections: + +* The `Transports`_ section documents asyncio :class:`BaseTransport`, + :class:`ReadTransport`, :class:`WriteTransport`, :class:`Transport`, + :class:`DatagramTransport`, and :class:`SubprocessTransport` + classes. + +* The `Protocols`_ section documents asyncio :class:`BaseProtocol`, + :class:`Protocol`, :class:`BufferedProtocol`, + :class:`DatagramProtocol`, and :class:`SubprocessProtocol` classes. + +* The `Examples`_ section showcases how to work with transports, + protocols, and low-level event loop APIs. -**Source code:** :source:`Lib/asyncio/protocols.py` .. _asyncio-transport: @@ -14,293 +70,360 @@ ========== Transports are classes provided by :mod:`asyncio` in order to abstract -various kinds of communication channels. You generally won't instantiate -a transport yourself; instead, you will call an :class:`AbstractEventLoop` method -which will create the transport and try to initiate the underlying -communication channel, calling you back when it succeeds. - -Once the communication channel is established, a transport is always -paired with a :ref:`protocol ` instance. The protocol can -then call the transport's methods for various purposes. - -:mod:`asyncio` currently implements transports for TCP, UDP, SSL, and -subprocess pipes. The methods available on a transport depend on -the transport's kind. +various kinds of communication channels. -The transport classes are :ref:`not thread safe `. +Transport objects are always instantiated by an +ref:`asyncio event loop `. -.. versionchanged:: 3.6 - The socket option ``TCP_NODELAY`` is now set by default. +asyncio implements transports for TCP, UDP, SSL, and subprocess pipes. +The methods available on a transport depend on the transport's kind. +The transport classes are :ref:`not thread safe `. -BaseTransport -------------- + +Transports Hierarchy +-------------------- .. class:: BaseTransport - Base class for transports. + Base class for all transports. Contains methods that all + asyncio transports share. - .. method:: close() +.. class:: WriteTransport(BaseTransport) - Close the transport. If the transport has a buffer for outgoing - data, buffered data will be flushed asynchronously. No more data - will be received. After all buffered data is flushed, the - protocol's :meth:`connection_lost` method will be called with - :const:`None` as its argument. + A base transport for write-only connections. - .. method:: is_closing() + Instances of the *WriteTransport* class are returned from + the :meth:`loop.connect_write_pipe` event loop method and + are also used by subprocess-related methods like + :meth:`loop.subprocess_exec`. - Return ``True`` if the transport is closing or is closed. +.. class:: ReadTransport(BaseTransport) - .. versionadded:: 3.5.1 + A base transport for read-only connections. - .. method:: get_extra_info(name, default=None) + Instances of the *ReadTransport* class are returned from + the :meth:`loop.connect_read_pipe` event loop method and + are also used by subprocess-related methods like + :meth:`loop.subprocess_exec`. - Return optional transport information. *name* is a string representing - the piece of transport-specific information to get, *default* is the - value to return if the information doesn't exist. +.. class:: Transport(WriteTransport, ReadTransport) - This method allows transport implementations to easily expose - channel-specific information. + Interface representing a bidirectional transport, such as a + TCP connection. - * socket: + The user does not instantiate a transport directly; they call a + utility function, passing it a protocol factory and other + information necessary to create the transport and protocol. - - ``'peername'``: the remote address to which the socket is connected, - result of :meth:`socket.socket.getpeername` (``None`` on error) - - ``'socket'``: :class:`socket.socket` instance - - ``'sockname'``: the socket's own address, - result of :meth:`socket.socket.getsockname` + Instances of the *Transport* class are returned from or used by + event loop methods like :meth:`loop.create_connection`, + :meth:`loop.create_unix_connection`, + :meth:`loop.create_server`, :meth:`loop.sendfile`, etc. - * SSL socket: - - ``'compression'``: the compression algorithm being used as a string, - or ``None`` if the connection isn't compressed; result of - :meth:`ssl.SSLSocket.compression` - - ``'cipher'``: a three-value tuple containing the name of the cipher - being used, the version of the SSL protocol that defines its use, and - the number of secret bits being used; result of - :meth:`ssl.SSLSocket.cipher` - - ``'peercert'``: peer certificate; result of - :meth:`ssl.SSLSocket.getpeercert` - - ``'sslcontext'``: :class:`ssl.SSLContext` instance - - ``'ssl_object'``: :class:`ssl.SSLObject` or :class:`ssl.SSLSocket` - instance +.. class:: DatagramTransport(BaseTransport) - * pipe: + A transport for datagram (UDP) connections. - - ``'pipe'``: pipe object + Instances of the *DatagramTransport* class are returned from + the :meth:`loop.create_datagram_endpoint` event loop method. - * subprocess: - - ``'subprocess'``: :class:`subprocess.Popen` instance +.. class:: SubprocessTransport(BaseTransport) - .. method:: set_protocol(protocol) + An abstraction to represent a connection between a parent and its + child OS process. - Set a new protocol. Switching protocol should only be done when both - protocols are documented to support the switch. + Instances of the *SubprocessTransport* class are returned from + event loop methods :meth:`loop.subprocess_shell` and + :meth:`loop.subprocess_exec`. - .. versionadded:: 3.5.3 - .. method:: get_protocol +Base Transport +-------------- - Return the current protocol. +.. method:: BaseTransport.close() - .. versionadded:: 3.5.3 + Close the transport. - .. versionchanged:: 3.5.1 - ``'ssl_object'`` info was added to SSL sockets. + If the transport has a buffer for outgoing + data, buffered data will be flushed asynchronously. No more data + will be received. After all buffered data is flushed, the + protocol's :meth:`protocol.connection_lost() + ` method will be called with + :const:`None` as its argument. +.. method:: BaseTransport.is_closing() -ReadTransport -------------- + Return ``True`` if the transport is closing or is closed. -.. class:: ReadTransport +.. method:: BaseTransport.get_extra_info(name, default=None) - Interface for read-only transports. + Return information about the transport or underlying resources + it uses. - .. method:: is_reading() + *name* is a string representing the piece of transport-specific + information to get. - Return ``True`` if the transport is receiving new data. + *default* is the value to return if the information is not + available, or if the transport does not support querying it + with the given third-party event loop implementation or on the + current platform. - .. versionadded:: 3.7 + For example, the following code attempts to get the underlying + socket object of the transport:: - .. method:: pause_reading() + sock = transport.get_extra_info('socket') + if sock is not None: + print(sock.getsockopt(...)) - Pause the receiving end of the transport. No data will be passed to - the protocol's :meth:`data_received` method until :meth:`resume_reading` - is called. + Categories of information that can be queried on some transports: - .. versionchanged:: 3.7 - The method is idempotent, i.e. it can be called when the - transport is already paused or closed. + * socket: - .. method:: resume_reading() + - ``'peername'``: the remote address to which the socket is + connected, result of :meth:`socket.socket.getpeername` + (``None`` on error) - Resume the receiving end. The protocol's :meth:`data_received` method - will be called once again if some data is available for reading. + - ``'socket'``: :class:`socket.socket` instance - .. versionchanged:: 3.7 - The method is idempotent, i.e. it can be called when the - transport is already reading. + - ``'sockname'``: the socket's own address, + result of :meth:`socket.socket.getsockname` + * SSL socket: -WriteTransport --------------- + - ``'compression'``: the compression algorithm being used as a + string, or ``None`` if the connection isn't compressed; result + of :meth:`ssl.SSLSocket.compression` + + - ``'cipher'``: a three-value tuple containing the name of the + cipher being used, the version of the SSL protocol that defines + its use, and the number of secret bits being used; result of + :meth:`ssl.SSLSocket.cipher` + + - ``'peercert'``: peer certificate; result of + :meth:`ssl.SSLSocket.getpeercert` -.. class:: WriteTransport + - ``'sslcontext'``: :class:`ssl.SSLContext` instance - Interface for write-only transports. + - ``'ssl_object'``: :class:`ssl.SSLObject` or + :class:`ssl.SSLSocket` instance - .. method:: abort() + * pipe: - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. + - ``'pipe'``: pipe object - .. method:: can_write_eof() + * subprocess: - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. + - ``'subprocess'``: :class:`subprocess.Popen` instance - .. method:: get_write_buffer_size() +.. method:: BaseTransport.set_protocol(protocol) - Return the current size of the output buffer used by the transport. + Set a new protocol. - .. method:: get_write_buffer_limits() + Switching protocol should only be done when both + protocols are documented to support the switch. - Get the *high*- and *low*-water limits for write flow control. Return a - tuple ``(low, high)`` where *low* and *high* are positive number of - bytes. +.. method:: BaseTransport.get_protocol() - Use :meth:`set_write_buffer_limits` to set the limits. + Return the current protocol. + + +Read-only Transports +-------------------- - .. versionadded:: 3.4.2 +.. method:: ReadTransport.is_reading() - .. method:: set_write_buffer_limits(high=None, low=None) + Return ``True`` if the transport is receiving new data. - Set the *high*- and *low*-water limits for write flow control. + .. versionadded:: 3.7 - These two values (measured in number of - bytes) control when the protocol's - :meth:`pause_writing` and :meth:`resume_writing` methods are called. - If specified, the low-water limit must be less than or equal to the - high-water limit. Neither *high* nor *low* can be negative. +.. method:: ReadTransport.pause_reading() - :meth:`pause_writing` is called when the buffer size becomes greater - than or equal to the *high* value. If writing has been paused, - :meth:`resume_writing` is called when the buffer size becomes less - than or equal to the *low* value. + Pause the receiving end of the transport. No data will be passed to + the protocol's :meth:`protocol.data_received() ` + method until :meth:`resume_reading` is called. - The defaults are implementation-specific. If only the - high-water limit is given, the low-water limit defaults to an - implementation-specific value less than or equal to the - high-water limit. Setting *high* to zero forces *low* to zero as - well, and causes :meth:`pause_writing` to be called whenever the - buffer becomes non-empty. Setting *low* to zero causes - :meth:`resume_writing` to be called only once the buffer is empty. - Use of zero for either limit is generally sub-optimal as it - reduces opportunities for doing I/O and computation - concurrently. + .. versionchanged:: 3.7 + The method is idempotent, i.e. it can be called when the + transport is already paused or closed. - Use :meth:`get_write_buffer_limits` to get the limits. +.. method:: ReadTransport.resume_reading() - .. method:: write(data) + Resume the receiving end. The protocol's + :meth:`protocol.data_received() ` method + will be called once again if some data is available for reading. - Write some *data* bytes to the transport. + .. versionchanged:: 3.7 + The method is idempotent, i.e. it can be called when the + transport is already reading. - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - .. method:: writelines(list_of_data) +Write-only Transports +--------------------- - Write a list (or any iterable) of data bytes to the transport. - This is functionally equivalent to calling :meth:`write` on each - element yielded by the iterable, but may be implemented more efficiently. +.. method:: WriteTransport.abort() - .. method:: write_eof() + Close the transport immediately, without waiting for pending operations + to complete. Buffered data will be lost. No more data will be received. + The protocol's :meth:`protocol.connection_lost() + ` method will eventually be + called with :const:`None` as its argument. - Close the write end of the transport after flushing buffered data. - Data may still be received. +.. method:: WriteTransport.can_write_eof() - This method can raise :exc:`NotImplementedError` if the transport - (e.g. SSL) doesn't support half-closes. + Return :const:`True` if the transport supports + :meth:`~WriteTransport.write_eof`, :const:`False` if not. +.. method:: WriteTransport.get_write_buffer_size() -DatagramTransport ------------------ + Return the current size of the output buffer used by the transport. -.. method:: DatagramTransport.sendto(data, addr=None) +.. method:: WriteTransport.get_write_buffer_limits() - Send the *data* bytes to the remote peer given by *addr* (a - transport-dependent target address). If *addr* is :const:`None`, the - data is sent to the target address given on transport creation. + Get the *high* and *low* watermarks for write flow control. Return a + tuple ``(low, high)`` where *low* and *high* are positive number of + bytes. + + Use :meth:`set_write_buffer_limits` to set the limits. + + .. versionadded:: 3.4.2 + +.. method:: WriteTransport.set_write_buffer_limits(high=None, low=None) + + Set the *high* and *low* watermarks for write flow control. + + These two values (measured in number of + bytes) control when the protocol's + :meth:`protocol.pause_writing() ` + and :meth:`protocol.resume_writing() ` + methods are called. If specified, the low watermark must be less + than or equal to the high watermark. Neither *high* nor *low* + can be negative. + + :meth:`~BaseProtocol.pause_writing` is called when the buffer size + becomes greater than or equal to the *high* value. If writing has + been paused, :meth:`~BaseProtocol.resume_writing` is called when + the buffer size becomes less than or equal to the *low* value. + + The defaults are implementation-specific. If only the + high watermark is given, the low watermark defaults to an + implementation-specific value less than or equal to the + high watermark. Setting *high* to zero forces *low* to zero as + well, and causes :meth:`~BaseProtocol.pause_writing` to be called + whenever the buffer becomes non-empty. Setting *low* to zero causes + :meth:`~BaseProtocol.resume_writing` to be called only once the + buffer is empty. Use of zero for either limit is generally + sub-optimal as it reduces opportunities for doing I/O and + computation concurrently. + + Use :meth:`~WriteTransport.get_write_buffer_limits` + to get the limits. + +.. method:: WriteTransport.write(data) + + Write some *data* bytes to the transport. This method does not block; it buffers the data and arranges for it to be sent out asynchronously. +.. method:: WriteTransport.writelines(list_of_data) + + Write a list (or any iterable) of data bytes to the transport. + This is functionally equivalent to calling :meth:`write` on each + element yielded by the iterable, but may be implemented more + efficiently. + +.. method:: WriteTransport.write_eof() + + Close the write end of the transport after flushing all buffered data. + Data may still be received. + + This method can raise :exc:`NotImplementedError` if the transport + (e.g. SSL) doesn't support half-closed connections. + + +Datagram Transports +------------------- + +.. method:: DatagramTransport.sendto(data, addr=None) + + Send the *data* bytes to the remote peer given by *addr* (a + transport-dependent target address). If *addr* is :const:`None`, + the data is sent to the target address given on transport + creation. + + This method does not block; it buffers the data and arranges + for it to be sent out asynchronously. + .. method:: DatagramTransport.abort() - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. + Close the transport immediately, without waiting for pending + operations to complete. Buffered data will be lost. + No more data will be received. The protocol's + :meth:`protocol.connection_lost() ` + method will eventually be called with :const:`None` as its argument. + +.. _asyncio-subprocess-transports: -BaseSubprocessTransport ------------------------ +Subprocess Transports +--------------------- -.. class:: BaseSubprocessTransport +.. method:: SubprocessTransport.get_pid() - .. method:: get_pid() + Return the subprocess process id as an integer. - Return the subprocess process id as an integer. +.. method:: SubprocessTransport.get_pipe_transport(fd) - .. method:: get_pipe_transport(fd) + Return the transport for the communication pipe corresponding to the + integer file descriptor *fd*: - Return the transport for the communication pipe corresponding to the - integer file descriptor *fd*: + * ``0``: readable streaming transport of the standard input (*stdin*), + or :const:`None` if the subprocess was not created with ``stdin=PIPE`` + * ``1``: writable streaming transport of the standard output (*stdout*), + or :const:`None` if the subprocess was not created with ``stdout=PIPE`` + * ``2``: writable streaming transport of the standard error (*stderr*), + or :const:`None` if the subprocess was not created with ``stderr=PIPE`` + * other *fd*: :const:`None` - * ``0``: readable streaming transport of the standard input (*stdin*), - or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), - or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), - or :const:`None` if the subprocess was not created with ``stderr=PIPE`` - * other *fd*: :const:`None` +.. method:: SubprocessTransport.get_returncode() - .. method:: get_returncode() + Return the subprocess return code as an integer or :const:`None` + if it hasn't returned, which is similar to the + :attr:`subprocess.Popen.returncode` attribute. - Return the subprocess returncode as an integer or :const:`None` - if it hasn't returned, similarly to the - :attr:`subprocess.Popen.returncode` attribute. +.. method:: SubprocessTransport.kill() - .. method:: kill() + Kill the subprocess. - Kill the subprocess, as in :meth:`subprocess.Popen.kill`. + On POSIX systems, the function sends SIGKILL to the subprocess. + On Windows, this method is an alias for :meth:`terminate`. - On POSIX systems, the function sends SIGKILL to the subprocess. - On Windows, this method is an alias for :meth:`terminate`. + See also :meth:`subprocess.Popen.kill`. - .. method:: send_signal(signal) +.. method:: SubprocessTransport.send_signal(signal) - Send the *signal* number to the subprocess, as in - :meth:`subprocess.Popen.send_signal`. + Send the *signal* number to the subprocess, as in + :meth:`subprocess.Popen.send_signal`. - .. method:: terminate() +.. method:: SubprocessTransport.terminate() - Ask the subprocess to stop, as in :meth:`subprocess.Popen.terminate`. - This method is an alias for the :meth:`close` method. + Stop the subprocess. - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to - stop the subprocess. + On POSIX systems, this method sends SIGTERM to the subprocess. + On Windows, the Windows API function TerminateProcess() is called to + stop the subprocess. - .. method:: close() + See also :meth:`subprocess.Popen.terminate`. - Ask the subprocess to stop by calling the :meth:`terminate` method if the - subprocess hasn't returned yet, and close transports of all pipes - (*stdin*, *stdout* and *stderr*). +.. method:: SubprocessTransport.close() + + Kill the subprocess by calling the :meth:`kill` method. + + If the subprocess hasn't returned yet, and close transports of + *stdin*, *stdout*, and *stderr* pipes. .. _asyncio-protocol: @@ -308,65 +431,61 @@ Protocols ========= -:mod:`asyncio` provides base classes that you can subclass to implement -your network protocols. Those classes are used in conjunction with -:ref:`transports ` (see below): the protocol parses incoming -data and asks for the writing of outgoing data, while the transport is -responsible for the actual I/O and buffering. - -When subclassing a protocol class, it is recommended you override certain -methods. Those methods are callbacks: they will be called by the transport -on certain events (for example when some data is received); you shouldn't -call them yourself, unless you are implementing a transport. +asyncio provides a set of abstract base classes that should be used +to implement network protocols. Those classes are meant to be used +together with :ref:`transports `. -.. note:: - All callbacks have default implementations, which are empty. Therefore, - you only need to implement the callbacks for the events in which you - are interested. +Subclasses of abstract base protocol classes may implement some or +all methods. All these methods are callbacks: they are called by +transports on certain events, for example when some data is received. +A base protocol method should be called by the corresponding transport. -Protocol classes ----------------- +Base Protocols +-------------- + +.. class:: BaseProtocol + + Base protocol with methods that all protocols share. -.. class:: Protocol +.. class:: Protocol(BaseProtocol) - The base class for implementing streaming protocols (for use with - e.g. TCP and SSL transports). + The base class for implementing streaming protocols + (TCP, Unix sockets, etc). -.. class:: BufferedProtocol +.. class:: BufferedProtocol(BaseProtocol) A base class for implementing streaming protocols with manual control of the receive buffer. - .. versionadded:: 3.7 - **Important:** this has been added to asyncio in Python 3.7 - *on a provisional basis*! Treat it as an experimental API that - might be changed or removed in Python 3.8. - -.. class:: DatagramProtocol +.. class:: DatagramProtocol(BaseProtocol) - The base class for implementing datagram protocols (for use with - e.g. UDP transports). + The base class for implementing datagram (UDP) protocols. -.. class:: SubprocessProtocol +.. class:: SubprocessProtocol(BaseProtocol) The base class for implementing protocols communicating with child - processes (through a set of unidirectional pipes). + processes (unidirectional pipes). -Connection callbacks --------------------- +Base Protocol +------------- + +All asyncio protocols can implement Base Protocol callbacks. + +.. rubric:: Connection Callbacks -These callbacks may be called on :class:`Protocol`, :class:`DatagramProtocol` -and :class:`SubprocessProtocol` instances: +Connection callbacks are called on all protocols, exactly once per +a successful connection. All other protocol callbacks can only be +called between those two methods. .. method:: BaseProtocol.connection_made(transport) Called when a connection is made. The *transport* argument is the transport representing the - connection. You are responsible for storing it somewhere - (e.g. as an attribute) if you need to. + connection. The protocol is responsible for storing the reference + to its transport. .. method:: BaseProtocol.connection_lost(exc) @@ -376,65 +495,76 @@ The latter means a regular EOF is received, or the connection was aborted or closed by this side of the connection. -:meth:`~BaseProtocol.connection_made` and :meth:`~BaseProtocol.connection_lost` -are called exactly once per successful connection. All other callbacks will be -called between those two methods, which allows for easier resource management -in your protocol implementation. -The following callbacks may be called only on :class:`SubprocessProtocol` -instances: +.. rubric:: Flow Control Callbacks -.. method:: SubprocessProtocol.pipe_data_received(fd, data) +Flow control callbacks can be called by transports to pause or +resume writing performed by the protocol. - Called when the child process writes data into its stdout or stderr pipe. - *fd* is the integer file descriptor of the pipe. *data* is a non-empty - bytes object containing the data. +See the documentation of the :meth:`~WriteTransport.set_write_buffer_limits` +method for more details. -.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc) +.. method:: BaseProtocol.pause_writing() - Called when one of the pipes communicating with the child process - is closed. *fd* is the integer file descriptor that was closed. + Called when the transport's buffer goes over the high watermark. -.. method:: SubprocessProtocol.process_exited() +.. method:: BaseProtocol.resume_writing() - Called when the child process has exited. + Called when the transport's buffer drains below the low watermark. + +If the buffer size equals the high watermark, +:meth:`~BaseProtocol.pause_writing` is not called: the buffer size must +go strictly over. +Conversely, :meth:`~BaseProtocol.resume_writing` is called when the +buffer size is equal or lower than the low watermark. These end +conditions are important to ensure that things go as expected when +either mark is zero. -Streaming protocols + +Streaming Protocols ------------------- -The following callbacks are called on :class:`Protocol` instances: +Event methods, such as :meth:`loop.create_server`, +:meth:`loop.create_unix_server`, :meth:`loop.create_connection`, +:meth:`loop.create_unix_connection`, :meth:`loop.connect_accepted_socket`, +:meth:`loop.connect_read_pipe`, and :meth:`loop.connect_write_pipe` +accept factories that return streaming protocols. .. method:: Protocol.data_received(data) - Called when some data is received. *data* is a non-empty bytes object - containing the incoming data. + Called when some data is received. *data* is a non-empty bytes + object containing the incoming data. - .. note:: - Whether the data is buffered, chunked or reassembled depends on - the transport. In general, you shouldn't rely on specific semantics - and instead make your parsing generic and flexible enough. However, - data is always received in the correct order. + Whether the data is buffered, chunked or reassembled depends on + the transport. In general, you shouldn't rely on specific semantics + and instead make your parsing generic and flexible. However, + data is always received in the correct order. + + The method can be called an arbitrary number of times while + a connection is open. + + However, :meth:`protocol.eof_received() ` + is called at most once. Once `eof_received()` is called, + ``data_received()`` is not called anymore. .. method:: Protocol.eof_received() Called when the other end signals it won't send any more data - (for example by calling :meth:`write_eof`, if the other end also uses + (for example by calling :meth:`transport.write_eof() + `, if the other end also uses asyncio). This method may return a false value (including ``None``), in which case the transport will close itself. Conversely, if this method returns a - true value, closing the transport is up to the protocol. Since the - default implementation returns ``None``, it implicitly closes the connection. + true value, the protocol used determines whether to close the transport. + Since the default implementation returns ``None``, it implicitly closes the + connection. + + Some transports, including SSL, don't support half-closed connections, + in which case returning true from this method will result in the connection + being closed. - .. note:: - Some transports such as SSL don't support half-closed connections, - in which case returning true from this method will not prevent closing - the connection. - -:meth:`data_received` can be called an arbitrary number of times during -a connection. However, :meth:`eof_received` is called at most once -and, if called, :meth:`data_received` won't be called after it. State machine: @@ -446,25 +576,23 @@ -> connection_lost -> end -Streaming protocols with manual receive buffer control ------------------------------------------------------- +Buffered Streaming Protocols +---------------------------- .. versionadded:: 3.7 - **Important:** :class:`BufferedProtocol` has been added to - asyncio in Python 3.7 *on a provisional basis*! Consider it as an - experimental API that might be changed or removed in Python 3.8. - + **Important:** this has been added to asyncio in Python 3.7 + *on a provisional basis*! This is as an experimental API that + might be changed or removed completely in Python 3.8. -Event methods, such as :meth:`AbstractEventLoop.create_server` and -:meth:`AbstractEventLoop.create_connection`, accept factories that -return protocols that implement this interface. +Buffered Protocols can be used with any event loop method +that supports `Streaming Protocols`_. -The idea of BufferedProtocol is that it allows to manually allocate -and control the receive buffer. Event loops can then use the buffer +``BufferedProtocol`` implementations allow explicit manual allocation +and control of the receive buffer. Event loops can then use the buffer provided by the protocol to avoid unnecessary data copies. This can result in noticeable performance improvement for protocols that -receive big amounts of data. Sophisticated protocols implementations -can allocate the buffer only once at creation time. +receive big amounts of data. Sophisticated protocol implementations +can significantly reduce the number of buffer allocations. The following callbacks are called on :class:`BufferedProtocol` instances: @@ -473,12 +601,12 @@ Called to allocate a new receive buffer. - *sizehint* is a recommended minimal size for the returned - buffer. It is acceptable to return smaller or bigger buffers + *sizehint* is the recommended minimum size for the returned + buffer. It is acceptable to return smaller or larger buffers than what *sizehint* suggests. When set to -1, the buffer size - can be arbitrary. It is an error to return a zero-sized buffer. + can be arbitrary. It is an error to return a buffer with a zero size. - Must return an object that implements the + ``get_buffer()`` must return an object implementing the :ref:`buffer protocol `. .. method:: BufferedProtocol.buffer_updated(nbytes) @@ -489,13 +617,15 @@ .. method:: BufferedProtocol.eof_received() - See the documentation of the :meth:`Protocol.eof_received` method. + See the documentation of the :meth:`protocol.eof_received() + ` method. -:meth:`get_buffer` can be called an arbitrary number of times during -a connection. However, :meth:`eof_received` is called at most once -and, if called, :meth:`get_buffer` and :meth:`buffer_updated` -won't be called after it. +:meth:`~BufferedProtocol.get_buffer` can be called an arbitrary number +of times during a connection. However, :meth:`protocol.eof_received() +` is called at most once +and, if called, :meth:`~BufferedProtocol.get_buffer` and +:meth:`~BufferedProtocol.buffer_updated` won't be called after it. State machine: @@ -509,10 +639,11 @@ -> connection_lost -> end -Datagram protocols +Datagram Protocols ------------------ -The following callbacks are called on :class:`DatagramProtocol` instances. +Datagram Protocol instances should be constructed by protocol +factories passed to the :meth:`loop.create_datagram_endpoint` method. .. method:: DatagramProtocol.datagram_received(data, addr) @@ -526,80 +657,120 @@ :class:`OSError`. *exc* is the :class:`OSError` instance. This method is called in rare conditions, when the transport (e.g. UDP) - detects that a datagram couldn't be delivered to its recipient. + detects that a datagram could not be delivered to its recipient. In many conditions though, undeliverable datagrams will be silently dropped. +.. note:: -Flow control callbacks ----------------------- + On BSD systems (macOS, FreeBSD, etc.) flow control is not supported + for datagram protocols, because there is no reliable way to detect send + failures caused by writing too many packets. + + The socket always appears 'ready' and excess packets are dropped. An + :class:`OSError` with ``errno`` set to :const:`errno.ENOBUFS` may + or may not be raised; if it is raised, it will be reported to + :meth:`DatagramProtocol.error_received` but otherwise ignored. -These callbacks may be called on :class:`Protocol`, -:class:`DatagramProtocol` and :class:`SubprocessProtocol` instances: -.. method:: BaseProtocol.pause_writing() +.. _asyncio-subprocess-protocols: - Called when the transport's buffer goes over the high-water mark. +Subprocess Protocols +-------------------- -.. method:: BaseProtocol.resume_writing() +Datagram Protocol instances should be constructed by protocol +factories passed to the :meth:`loop.subprocess_exec` and +:meth:`loop.subprocess_shell` methods. - Called when the transport's buffer drains below the low-water mark. +.. method:: SubprocessProtocol.pipe_data_received(fd, data) + Called when the child process writes data into its stdout or stderr + pipe. -:meth:`pause_writing` and :meth:`resume_writing` calls are paired -- -:meth:`pause_writing` is called once when the buffer goes strictly over -the high-water mark (even if subsequent writes increases the buffer size -even more), and eventually :meth:`resume_writing` is called once when the -buffer size reaches the low-water mark. + *fd* is the integer file descriptor of the pipe. -.. note:: - If the buffer size equals the high-water mark, - :meth:`pause_writing` is not called -- it must go strictly over. - Conversely, :meth:`resume_writing` is called when the buffer size is - equal or lower than the low-water mark. These end conditions - are important to ensure that things go as expected when either - mark is zero. + *data* is a non-empty bytes object containing the received data. + +.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc) + + Called when one of the pipes communicating with the child process + is closed. + + *fd* is the integer file descriptor that was closed. + +.. method:: SubprocessProtocol.process_exited() + + Called when the child process has exited. + + +Examples +======== + +.. _asyncio_example_tcp_echo_server_protocol: + +TCP Echo Server +--------------- + +Create a TCP echo server using the :meth:`loop.create_server` method, send back +received data, and close the connection:: + + import asyncio + + + class EchoServerClientProtocol(asyncio.Protocol): + def connection_made(self, transport): + peername = transport.get_extra_info('peername') + print('Connection from {}'.format(peername)) + self.transport = transport + + def data_received(self, data): + message = data.decode() + print('Data received: {!r}'.format(message)) + + print('Send: {!r}'.format(message)) + self.transport.write(data) + + print('Close the client socket') + self.transport.close() -.. note:: - On BSD systems (OS X, FreeBSD, etc.) flow control is not supported - for :class:`DatagramProtocol`, because send failures caused by - writing too many packets cannot be detected easily. The socket - always appears 'ready' and excess packets are dropped; an - :class:`OSError` with errno set to :const:`errno.ENOBUFS` may or - may not be raised; if it is raised, it will be reported to - :meth:`DatagramProtocol.error_received` but otherwise ignored. + async def main(): + # Get a reference to the event loop as we plan to use + # low-level APIs. + loop = asyncio.get_running_loop() -Coroutines and protocols ------------------------- + server = await loop.create_server( + lambda: EchoServerClientProtocol(), + '127.0.0.1', 8888) -Coroutines can be scheduled in a protocol method using :func:`ensure_future`, -but there is no guarantee made about the execution order. Protocols are not -aware of coroutines created in protocol methods and so will not wait for them. + async with server: + await server.serve_forever() -To have a reliable execution order, -use :ref:`stream objects ` in a -coroutine with ``await``. For example, the :meth:`StreamWriter.drain` -coroutine can be used to wait until the write buffer is flushed. + asyncio.run(main()) + + +.. seealso:: -Protocol examples -================= + The :ref:`TCP echo server using streams ` + example uses the high-level :func:`asyncio.start_server` function. -.. _asyncio-tcp-echo-client-protocol: +.. _asyncio_example_tcp_echo_client_protocol: -TCP echo client protocol ------------------------- +TCP Echo Client +--------------- -TCP echo client using the :meth:`AbstractEventLoop.create_connection` method, send -data and wait until the connection is closed:: +A TCP echo client using the :meth:`loop.create_connection` method, sends +data, and waits until the connection is closed:: import asyncio + class EchoClientProtocol(asyncio.Protocol): - def __init__(self, message, loop): + def __init__(self, message, on_con_lost, loop): self.message = message self.loop = loop + self.on_con_lost = on_con_lost def connection_made(self, transport): transport.write(self.message.encode()) @@ -610,99 +781,99 @@ def connection_lost(self, exc): print('The server closed the connection') - print('Stop the event loop') - self.loop.stop() + self.on_con_lost.set_result(True) + + + async def main(): + # Get a reference to the event loop as we plan to use + # low-level APIs. + loop = asyncio.get_running_loop() + + on_con_lost = loop.create_future() + message = 'Hello World!' + + transport, protocol = await loop.create_connection( + lambda: EchoClientProtocol(message, on_con_lost, loop), + '127.0.0.1', 8888) + + # Wait until the protocol signals that the connection + # is lost and close the transport. + try: + await on_con_lost + finally: + transport.close() + + + asyncio.run(main()) - loop = asyncio.get_event_loop() - message = 'Hello World!' - coro = loop.create_connection(lambda: EchoClientProtocol(message, loop), - '127.0.0.1', 8888) - loop.run_until_complete(coro) - loop.run_forever() - loop.close() - -The event loop is running twice. The -:meth:`~AbstractEventLoop.run_until_complete` method is preferred in this short -example to raise an exception if the server is not listening, instead of -having to write a short coroutine to handle the exception and stop the -running loop. At :meth:`~AbstractEventLoop.run_until_complete` exit, the loop is -no longer running, so there is no need to stop the loop in case of an error. .. seealso:: The :ref:`TCP echo client using streams ` - example uses the :func:`asyncio.open_connection` function. + example uses the high-level :func:`asyncio.open_connection` function. -.. _asyncio-tcp-echo-server-protocol: +.. _asyncio-udp-echo-server-protocol: -TCP echo server protocol ------------------------- +UDP Echo Server +--------------- -TCP echo server using the :meth:`AbstractEventLoop.create_server` method, send back -received data and close the connection:: +A UDP echo server, using the :meth:`loop.create_datagram_endpoint` +method, sends back received data:: import asyncio - class EchoServerClientProtocol(asyncio.Protocol): + + class EchoServerProtocol: def connection_made(self, transport): - peername = transport.get_extra_info('peername') - print('Connection from {}'.format(peername)) self.transport = transport - def data_received(self, data): + def datagram_received(self, data, addr): message = data.decode() - print('Data received: {!r}'.format(message)) + print('Received %r from %s' % (message, addr)) + print('Send %r to %s' % (message, addr)) + self.transport.sendto(data, addr) - print('Send: {!r}'.format(message)) - self.transport.write(data) - print('Close the client socket') - self.transport.close() + async def main(): + print("Starting UDP server") - loop = asyncio.get_event_loop() - # Each client connection will create a new protocol instance - coro = loop.create_server(EchoServerClientProtocol, '127.0.0.1', 8888) - server = loop.run_until_complete(coro) - - # Serve requests until Ctrl+C is pressed - print('Serving on {}'.format(server.sockets[0].getsockname())) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - # Close the server - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - -:meth:`Transport.close` can be called immediately after -:meth:`WriteTransport.write` even if data are not sent yet on the socket: both -methods are asynchronous. ``await`` is not needed because these transport -methods are not coroutines. + # Get a reference to the event loop as we plan to use + # low-level APIs. + loop = asyncio.get_running_loop() + + # One protocol instance will be created to serve all + # client requests. + transport, protocol = await loop.create_datagram_endpoint( + lambda: EchoServerProtocol(), + local_addr=('127.0.0.1', 9999)) + + try: + await asyncio.sleep(3600) # Serve for 1 hour. + finally: + transport.close() -.. seealso:: - The :ref:`TCP echo server using streams ` - example uses the :func:`asyncio.start_server` function. + asyncio.run(main()) .. _asyncio-udp-echo-client-protocol: -UDP echo client protocol ------------------------- +UDP Echo Client +--------------- -UDP echo client using the :meth:`AbstractEventLoop.create_datagram_endpoint` -method, send data and close the transport when we received the answer:: +A UDP echo client, using the :meth:`loop.create_datagram_endpoint` +method, sends data and closes the transport when it receives the answer:: import asyncio + class EchoClientProtocol: def __init__(self, message, loop): self.message = message self.loop = loop self.transport = None + self.on_con_lost = loop.create_future() def connection_made(self, transport): self.transport = transport @@ -719,75 +890,46 @@ print('Error received:', exc) def connection_lost(self, exc): - print("Socket closed, stop the event loop") - loop = asyncio.get_event_loop() - loop.stop() - - loop = asyncio.get_event_loop() - message = "Hello World!" - connect = loop.create_datagram_endpoint( - lambda: EchoClientProtocol(message, loop), - remote_addr=('127.0.0.1', 9999)) - transport, protocol = loop.run_until_complete(connect) - loop.run_forever() - transport.close() - loop.close() + print("Connection closed") + self.on_con_lost.set_result(True) -.. _asyncio-udp-echo-server-protocol: + async def main(): + # Get a reference to the event loop as we plan to use + # low-level APIs. + loop = asyncio.get_running_loop() -UDP echo server protocol ------------------------- + message = "Hello World!" + transport, protocol = await loop.create_datagram_endpoint( + lambda: EchoClientProtocol(message, loop), + remote_addr=('127.0.0.1', 9999)) -UDP echo server using the :meth:`AbstractEventLoop.create_datagram_endpoint` -method, send back received data:: + try: + await protocol.on_con_lost + finally: + transport.close() - import asyncio - class EchoServerProtocol: - def connection_made(self, transport): - self.transport = transport - - def datagram_received(self, data, addr): - message = data.decode() - print('Received %r from %s' % (message, addr)) - print('Send %r to %s' % (message, addr)) - self.transport.sendto(data, addr) - - loop = asyncio.get_event_loop() - print("Starting UDP server") - # One protocol instance will be created to serve all client requests - listen = loop.create_datagram_endpoint( - EchoServerProtocol, local_addr=('127.0.0.1', 9999)) - transport, protocol = loop.run_until_complete(listen) - - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - transport.close() - loop.close() + asyncio.run(main()) -.. _asyncio-register-socket: +.. _asyncio_example_create_connection: -Register an open socket to wait for data using a protocol ---------------------------------------------------------- +Connecting Existing Sockets +--------------------------- Wait until a socket receives data using the -:meth:`AbstractEventLoop.create_connection` method with a protocol, and then close -the event loop :: +:meth:`loop.create_connection` method with a protocol:: import asyncio - from socket import socketpair + import socket - # Create a pair of connected sockets - rsock, wsock = socketpair() - loop = asyncio.get_event_loop() class MyProtocol(asyncio.Protocol): - transport = None + + def __init__(self, loop): + self.transport = None + self.on_con_lost = loop.create_future() def connection_made(self, transport): self.transport = transport @@ -795,35 +937,105 @@ def data_received(self, data): print("Received:", data.decode()) - # We are done: close the transport (it will call connection_lost()) + # We are done: close the transport; + # connection_lost() will be called automatically. self.transport.close() def connection_lost(self, exc): - # The socket has been closed, stop the event loop - loop.stop() + # The socket has been closed + self.on_con_lost.set_result(True) + + + async def main(): + # Get a reference to the event loop as we plan to use + # low-level APIs. + loop = asyncio.get_running_loop() + + # Create a pair of connected sockets + rsock, wsock = socket.socketpair() + + # Register the socket to wait for data. + transport, protocol = await loop.create_connection( + lambda: MyProtocol(loop), sock=rsock) - # Register the socket to wait for data - connect_coro = loop.create_connection(MyProtocol, sock=rsock) - transport, protocol = loop.run_until_complete(connect_coro) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Run the event loop - loop.run_forever() - - # We are done, close sockets and the event loop - rsock.close() - wsock.close() - loop.close() + # Simulate the reception of data from the network. + loop.call_soon(wsock.send, 'abc'.encode()) + + try: + await protocol.on_con_lost + finally: + transport.close() + wsock.close() + + asyncio.run(main()) .. seealso:: The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`AbstractEventLoop.add_reader` method to register the file descriptor of a - socket. + ` example uses the low-level + :meth:`loop.add_reader` method to register an FD. The :ref:`register an open socket to wait for data using streams - ` example uses high-level streams + ` example uses high-level streams created by the :func:`open_connection` function in a coroutine. + +.. _asyncio_example_subprocess_proto: + +loop.subprocess_exec() and SubprocessProtocol +--------------------------------------------- + +An example of a subprocess protocol used to get the output of a +subprocess and to wait for the subprocess exit. + +The subprocess is created by th :meth:`loop.subprocess_exec` method:: + + import asyncio + import sys + + class DateProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future): + self.exit_future = exit_future + self.output = bytearray() + + def pipe_data_received(self, fd, data): + self.output.extend(data) + + def process_exited(self): + self.exit_future.set_result(True) + + async def get_date(): + # Get a reference to the event loop as we plan to use + # low-level APIs. + loop = asyncio.get_running_loop() + + code = 'import datetime; print(datetime.datetime.now())' + exit_future = asyncio.Future(loop=loop) + + # Create the subprocess controlled by DateProtocol; + # redirect the standard output into a pipe. + transport, protocol = await loop.subprocess_exec( + lambda: DateProtocol(exit_future), + sys.executable, '-c', code, + stdin=None, stderr=None) + + # Wait for the subprocess exit using the process_exited() + # method of the protocol. + await exit_future + + # Close the stdout pipe. + transport.close() + + # Read the output which was collected by the + # pipe_data_received() method of the protocol. + data = bytes(protocol.output) + return data.decode('ascii').rstrip() + + if sys.platform == "win32": + asyncio.set_event_loop_policy( + asyncio.WindowsProactorEventLoopPolicy()) + + date = asyncio.run(get_date()) + print(f"Current date: {date}") + +See also the :ref:`same example ` +written using high-level APIs. diff -Nru python3.7-3.7.0/Doc/library/asyncio-queue.rst python3.7-3.7.1/Doc/library/asyncio-queue.rst --- python3.7-3.7.0/Doc/library/asyncio-queue.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-queue.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,41 +1,42 @@ .. currentmodule:: asyncio +.. _asyncio-queues: + +====== Queues ====== -**Source code:** :source:`Lib/asyncio/queues.py` - -Queues: +asyncio queues are designed to be similar to classes of the +:mod:`queue` module. Although asyncio queues are not thread-safe, +they are designed to be used specifically in async/await code. + +Note that methods of asyncio queues don't have a *timeout* parameter; +use :func:`asyncio.wait_for` function to do queue operations with a +timeout. -* :class:`Queue` -* :class:`PriorityQueue` -* :class:`LifoQueue` - -asyncio queue API was designed to be close to classes of the :mod:`queue` -module (:class:`~queue.Queue`, :class:`~queue.PriorityQueue`, -:class:`~queue.LifoQueue`), but it has no *timeout* parameter. The -:func:`asyncio.wait_for` function can be used to cancel a task after a timeout. +See also the `Examples`_ section below. Queue ------ +===== .. class:: Queue(maxsize=0, \*, loop=None) - A queue, useful for coordinating producer and consumer coroutines. + A first in, first out (FIFO) queue. - If *maxsize* is less than or equal to zero, the queue size is infinite. If - it is an integer greater than ``0``, then ``await put()`` will block - when the queue reaches *maxsize*, until an item is removed by :meth:`get`. - - Unlike the standard library :mod:`queue`, you can reliably know this Queue's - size with :meth:`qsize`, since your single-threaded asyncio application won't - be interrupted between calling :meth:`qsize` and doing an operation on the - Queue. + If *maxsize* is less than or equal to zero, the queue size is + infinite. If it is an integer greater than ``0``, then + ``await put()`` blocks when the queue reaches *maxsize* + until an item is removed by :meth:`get`. + + Unlike the standard library threading :mod:`queue`, the size of + the queue is always known and can be returned by calling the + :meth:`qsize` method. This class is :ref:`not thread safe `. - .. versionchanged:: 3.4.4 - New :meth:`join` and :meth:`task_done` methods. + .. attribute:: maxsize + + Number of items allowed in the queue. .. method:: empty() @@ -45,53 +46,33 @@ Return ``True`` if there are :attr:`maxsize` items in the queue. - .. note:: - - If the Queue was initialized with ``maxsize=0`` (the default), then - :meth:`full()` is never ``True``. + If the queue was initialized with ``maxsize=0`` (the default), + then :meth:`full()` never returns ``True``. .. coroutinemethod:: get() - Remove and return an item from the queue. If queue is empty, wait until - an item is available. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`empty` method. + Remove and return an item from the queue. If queue is empty, + wait until an item is available. .. method:: get_nowait() - Remove and return an item from the queue. - Return an item if one is immediately available, else raise :exc:`QueueEmpty`. .. coroutinemethod:: join() - Block until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer thread calls - :meth:`task_done` to indicate that the item was retrieved and all work on - it is complete. When the count of unfinished tasks drops to zero, - :meth:`join` unblocks. - - This method is a :ref:`coroutine `. + Block until all items in the queue have been received and processed. - .. versionadded:: 3.4.4 + The count of unfinished tasks goes up whenever an item is added + to the queue. The count goes down whenever a consumer thread calls + :meth:`task_done` to indicate that the item was retrieved and all + work on it is complete. When the count of unfinished tasks drops + to zero, :meth:`join` unblocks. .. coroutinemethod:: put(item) - Put an item into the queue. If the queue is full, wait until a free slot - is available before adding item. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`full` method. + Put an item into the queue. If the queue is full, wait until a + free slot is available before adding the item. .. method:: put_nowait(item) @@ -101,60 +82,119 @@ .. method:: qsize() - Number of items in the queue. + Return the number of items in the queue. .. method:: task_done() Indicate that a formerly enqueued task is complete. - Used by queue consumers. For each :meth:`~Queue.get` used to fetch a task, a - subsequent call to :meth:`task_done` tells the queue that the processing - on the task is complete. - - If a :meth:`join` is currently blocking, it will resume when all items - have been processed (meaning that a :meth:`task_done` call was received - for every item that had been :meth:`~Queue.put` into the queue). - - Raises :exc:`ValueError` if called more times than there were items - placed in the queue. + Used by queue consumers. For each :meth:`~Queue.get` used to + fetch a task, a subsequent call to :meth:`task_done` tells the + queue that the processing on the task is complete. - .. versionadded:: 3.4.4 + If a :meth:`join` is currently blocking, it will resume when all + items have been processed (meaning that a :meth:`task_done` + call was received for every item that had been :meth:`~Queue.put` + into the queue). - .. attribute:: maxsize - - Number of items allowed in the queue. + Raises :exc:`ValueError` if called more times than there were + items placed in the queue. -PriorityQueue -------------- +Priority Queue +============== .. class:: PriorityQueue - A subclass of :class:`Queue`; retrieves entries in priority order (lowest - first). + A variant of :class:`Queue`; retrieves entries in priority order + (lowest first). - Entries are typically tuples of the form: (priority number, data). + Entries are typically tuples of the form + ``(priority_number, data)``. -LifoQueue ---------- +LIFO Queue +========== .. class:: LifoQueue - A subclass of :class:`Queue` that retrieves most recently added entries - first. + A variant of :class:`Queue` that retrieves most recently added + entries first (last in, first out). Exceptions -^^^^^^^^^^ +========== .. exception:: QueueEmpty - Exception raised when the :meth:`~Queue.get_nowait` method is called on a - :class:`Queue` object which is empty. + This exception is raised when the :meth:`~Queue.get_nowait` method + is called on an empty queue. .. exception:: QueueFull - Exception raised when the :meth:`~Queue.put_nowait` method is called on a - :class:`Queue` object which is full. + Exception raised when the :meth:`~Queue.put_nowait` method is called + on a queue that has reached its *maxsize*. + + +Examples +======== + +.. _asyncio_example_queue_dist: + +Queues can be used to distribute workload between several +concurrent tasks:: + + import asyncio + import random + import time + + + async def worker(name, queue): + while True: + # Get a "work item" out of the queue. + sleep_for = await queue.get() + + # Sleep for the "sleep_for" seconds. + await asyncio.sleep(sleep_for) + + # Notify the queue that the "work item" has been processed. + queue.task_done() + + print(f'{name} has slept for {sleep_for:.2f} seconds') + + + async def main(): + # Create a queue that we will use to store our "workload". + queue = asyncio.Queue() + + # Generate random timings and put them into the queue. + total_sleep_time = 0 + for _ in range(20): + sleep_for = random.uniform(0.05, 1.0) + total_sleep_time += sleep_for + queue.put_nowait(sleep_for) + + # Create three worker tasks to process the queue concurrently. + tasks = [] + for i in range(3): + task = asyncio.create_task(worker(f'worker-{i}', queue)) + tasks.append(task) + + # Wait until the queue is fully processed. + started_at = time.monotonic() + await queue.join() + total_slept_for = time.monotonic() - started_at + + # Cancel our worker tasks. + for task in tasks: + task.cancel() + # Wait until all worker tasks are cancelled. + await asyncio.gather(*tasks, return_exceptions=True) + + print('====') + print(f'3 workers slept in parallel for {total_slept_for:.2f} seconds') + print(f'total expected sleep time: {total_sleep_time:.2f} seconds') + + + asyncio.run(main()) diff -Nru python3.7-3.7.0/Doc/library/asyncio.rst python3.7-3.7.1/Doc/library/asyncio.rst --- python3.7-3.7.0/Doc/library/asyncio.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,66 +1,93 @@ -:mod:`asyncio` --- Asynchronous I/O, event loop, coroutines and tasks -===================================================================== +:mod:`asyncio` --- Asynchronous I/O +=================================== .. module:: asyncio - :synopsis: Asynchronous I/O, event loop, coroutines and tasks. + :synopsis: Asynchronous I/O. -.. versionadded:: 3.4 +-------------- -**Source code:** :source:`Lib/asyncio/` +.. sidebar:: Hello World! --------------- + :: + + import asyncio + + async def main(): + print('Hello ...') + await asyncio.sleep(1) + print('... World!') + + # Python 3.7+ + asyncio.run(main()) + +asyncio is a library to write **concurrent** code using +the **async/await** syntax. -This module provides infrastructure for writing single-threaded concurrent -code using coroutines, multiplexing I/O access over sockets and other -resources, running network clients and servers, and other related primitives. -Here is a more detailed list of the package contents: +asyncio is used as a foundation for multiple Python asynchronous +frameworks that provide high-performance network and web-servers, +database connection libraries, distributed task queues, etc. -* a pluggable :ref:`event loop ` with various system-specific - implementations; +asyncio is often a perfect fit for IO-bound and high-level +**structured** network code. -* :ref:`transport ` and :ref:`protocol ` abstractions - (similar to those in `Twisted `_); +asyncio provides a set of **high-level** APIs to: -* concrete support for TCP, UDP, SSL, subprocess pipes, delayed calls, and - others (some may be system-dependent); +* :ref:`run Python coroutines ` concurrently and + have full control over their execution; -* a :class:`Future` class that mimics the one in the :mod:`concurrent.futures` - module, but adapted for use with the event loop; +* perform :ref:`network IO and IPC `; -* coroutines and tasks based on ``yield from`` (:PEP:`380`), to help write - concurrent code in a sequential fashion; +* control :ref:`subprocesses `; -* cancellation support for :class:`Future`\s and coroutines; +* distribute tasks via :ref:`queues `; -* :ref:`synchronization primitives ` for use between coroutines in - a single thread, mimicking those in the :mod:`threading` module; +* :ref:`synchronize ` concurrent code; -* an interface for passing work off to a threadpool, for times when - you absolutely, positively have to use a library that makes blocking - I/O calls. +Additionally, there are **low-level** APIs for +*library and framework developers* to: -Asynchronous programming is more complex than classical "sequential" -programming: see the :ref:`Develop with asyncio ` page which lists -common traps and explains how to avoid them. :ref:`Enable the debug mode -` during development to detect common issues. +* create and manage :ref:`event loops `, which + provide asynchronous APIs for :meth:`networking `, + running :meth:`subprocesses `, + handling :meth:`OS signals `, etc; -Table of contents: +* implement efficient protocols using + :ref:`transports `; + +* :ref:`bridge ` callback-based libraries and code + with async/await syntax. + + +.. We use the "rubric" directive here to avoid creating + the "Reference" subsection in the TOC. + +.. rubric:: Reference .. toctree:: - :maxdepth: 3 + :caption: High-level APIs + :maxdepth: 1 - asyncio-eventloop.rst - asyncio-eventloops.rst asyncio-task.rst - asyncio-protocol.rst asyncio-stream.rst - asyncio-subprocess.rst asyncio-sync.rst + asyncio-subprocess.rst asyncio-queue.rst - asyncio-dev.rst + asyncio-exceptions.rst + +.. toctree:: + :caption: Low-level APIs + :maxdepth: 1 -.. seealso:: + asyncio-eventloop.rst + asyncio-future.rst + asyncio-protocol.rst + asyncio-policy.rst + asyncio-platforms.rst - The :mod:`asyncio` module was designed in :PEP:`3156`. For a - motivational primer on transports and protocols, see :PEP:`3153`. +.. toctree:: + :caption: Guides and Tutorials + :maxdepth: 1 + asyncio-api-index.rst + asyncio-llapi-index.rst + asyncio-dev.rst diff -Nru python3.7-3.7.0/Doc/library/asyncio-stream.rst python3.7-3.7.1/Doc/library/asyncio-stream.rst --- python3.7-3.7.0/Doc/library/asyncio-stream.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-stream.rst 2018-10-20 06:04:19.000000000 +0000 @@ -2,85 +2,120 @@ .. _asyncio-streams: -+++++++++++++++++++++++++++++ -Streams (coroutine based API) -+++++++++++++++++++++++++++++ +======= +Streams +======= + +Streams are high-level async/await-ready primitives to work with +network connections. Streams allow sending and receiving data without +using callbacks or low-level protocols and transports. -**Source code:** :source:`Lib/asyncio/streams.py` +.. _asyncio_example_stream: -Stream functions -================ +Here is an example of a TCP echo client written using asyncio +streams:: -.. note:: + import asyncio + + async def tcp_echo_client(message): + reader, writer = await asyncio.open_connection( + '127.0.0.1', 8888) - The top-level functions in this module are meant as convenience wrappers - only; there's really nothing special there, and if they don't do - exactly what you want, feel free to copy their code. + print(f'Send: {message!r}') + writer.write(message.encode()) + data = await reader.read(100) + print(f'Received: {data.decode()!r}') -.. coroutinefunction:: open_connection(host=None, port=None, \*, loop=None, limit=None, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None, ssl_handshake_timeout=None) + print('Close the connection') + writer.close() + await writer.wait_closed() - A wrapper for :meth:`~AbstractEventLoop.create_connection()` returning a (reader, - writer) pair. + asyncio.run(tcp_echo_client('Hello World!')) - The reader returned is a :class:`StreamReader` instance; the writer is - a :class:`StreamWriter` instance. - When specified, the *loop* argument determines which event loop to use, - and the *limit* argument determines the buffer size limit used by the - returned :class:`StreamReader` instance. +See also the `Examples`_ section below. - The rest of the arguments are passed directly to - :meth:`AbstractEventLoop.create_connection`. - This function is a :ref:`coroutine `. +.. rubric:: Stream Functions + +The following top-level asyncio functions can be used to create +and work with streams: + + +.. coroutinefunction:: open_connection(host=None, port=None, \*, \ + loop=None, limit=None, ssl=None, family=0, \ + proto=0, flags=0, sock=None, local_addr=None, \ + server_hostname=None, ssl_handshake_timeout=None) + + Establish a network connection and return a pair of + ``(reader, writer)`` objects. + + The returned *reader* and *writer* objects are instances of + :class:`StreamReader` and :class:`StreamWriter` classes. + + The *loop* argument is optional and can always be determined + automatically when this function is awaited from a coroutine. + + *limit* determines the buffer size limit used by the + returned :class:`StreamReader` instance. By default the *limit* + is set to 64 KiB. + + The rest of the arguments are passed directly to + :meth:`loop.create_connection`. .. versionadded:: 3.7 The *ssl_handshake_timeout* parameter. -.. coroutinefunction:: start_server(client_connected_cb, host=None, port=None, \*, loop=None, limit=None, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, ssl_handshake_timeout=None, start_serving=True) +.. coroutinefunction:: start_server(client_connected_cb, host=None, \ + port=None, \*, loop=None, limit=None, \ + family=socket.AF_UNSPEC, \ + flags=socket.AI_PASSIVE, sock=None, \ + backlog=100, ssl=None, reuse_address=None, \ + reuse_port=None, ssl_handshake_timeout=None, \ + start_serving=True) - Start a socket server, with a callback for each client connected. The return - value is the same as :meth:`~AbstractEventLoop.create_server()`. + Start a socket server. The *client_connected_cb* callback is called whenever a new client - connection is established. It receives a reader/writer pair as two - arguments, the first is a :class:`StreamReader` instance, - and the second is a :class:`StreamWriter` instance. + connection is established. It receives a ``(reader, writer)`` pair + as two arguments, instances of the :class:`StreamReader` and + :class:`StreamWriter` classes. - *client_connected_cb* accepts a plain callable or a + *client_connected_cb* can be a plain callable or a :ref:`coroutine function `; if it is a coroutine function, - it will be automatically converted into a :class:`Task`. + it will be automatically scheduled as a :class:`Task`. - When specified, the *loop* argument determines which event loop to use, - and the *limit* argument determines the buffer size limit used by the - :class:`StreamReader` instance passed to *client_connected_cb*. + The *loop* argument is optional and can always be determined + automatically when this method is awaited from a coroutine. - The rest of the arguments are passed directly to - :meth:`~AbstractEventLoop.create_server()`. + *limit* determines the buffer size limit used by the + returned :class:`StreamReader` instance. By default the *limit* + is set to 64 KiB. - This function is a :ref:`coroutine `. + The rest of the arguments are passed directly to + :meth:`loop.create_server`. .. versionadded:: 3.7 The *ssl_handshake_timeout* and *start_serving* parameters. -.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, limit=None, ssl=None, sock=None, server_hostname=None, ssl_handshake_timeout=None) - A wrapper for :meth:`~AbstractEventLoop.create_unix_connection()` returning - a (reader, writer) pair. +.. rubric:: Unix Sockets - When specified, the *loop* argument determines which event loop to use, - and the *limit* argument determines the buffer size limit used by the - returned :class:`StreamReader` instance. +.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \ + limit=None, ssl=None, sock=None, \ + server_hostname=None, ssl_handshake_timeout=None) - The rest of the arguments are passed directly to - :meth:`~AbstractEventLoop.create_unix_connection()`. + Establish a Unix socket connection and return a pair of + ``(reader, writer)``. - This function is a :ref:`coroutine `. + Similar to :func:`open_connection` but operates on Unix sockets. - Availability: UNIX. + See also the documentation of :meth:`loop.create_unix_connection`. + + Availability: Unix. .. versionadded:: 3.7 @@ -90,29 +125,19 @@ The *path* parameter can now be a :term:`path-like object` -.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \*, loop=None, limit=None, sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, start_serving=True) - Start a UNIX Domain Socket server, with a callback for each client connected. +.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ + \*, loop=None, limit=None, sock=None, \ + backlog=100, ssl=None, ssl_handshake_timeout=None, \ + start_serving=True) - The *client_connected_cb* callback is called whenever a new client - connection is established. It receives a reader/writer pair as two - arguments, the first is a :class:`StreamReader` instance, - and the second is a :class:`StreamWriter` instance. + Start a Unix socket server. - *client_connected_cb* accepts a plain callable or a - :ref:`coroutine function `; if it is a coroutine function, - it will be automatically converted into a :class:`Task`. + Similar to :func:`start_server` but works with Unix sockets. - When specified, the *loop* argument determines which event loop to use, - and the *limit* argument determines the buffer size limit used by the - :class:`StreamReader` instance passed to *client_connected_cb*. - - The rest of the arguments are passed directly to - :meth:`~AbstractEventLoop.create_unix_server()`. + See also the documentation of :meth:`loop.create_unix_server`. - This function is a :ref:`coroutine `. - - Availability: UNIX. + Availability: Unix. .. versionadded:: 3.7 @@ -123,227 +148,156 @@ The *path* parameter can now be a :term:`path-like object`. -StreamReader -============ - -.. class:: StreamReader(limit=None, loop=None) - - This class is :ref:`not thread safe `. - - .. method:: exception() +--------- - Get the exception. - .. method:: feed_eof() - - Acknowledge the EOF. - - .. method:: feed_data(data) - - Feed *data* bytes in the internal buffer. Any operations waiting - for the data will be resumed. - - .. method:: set_exception(exc) +StreamReader +============ - Set the exception. +.. class:: StreamReader - .. method:: set_transport(transport) + Represents a reader object that provides APIs to read data + from the IO stream. - Set the transport. + It is not recommended to instantiate *StreamReader* objects + directly; use :func:`open_connection` and :func:`start_server` + instead. .. coroutinemethod:: read(n=-1) Read up to *n* bytes. If *n* is not provided, or set to ``-1``, read until EOF and return all read bytes. - If the EOF was received and the internal buffer is empty, + If EOF was received and the internal buffer is empty, return an empty ``bytes`` object. - This method is a :ref:`coroutine `. - .. coroutinemethod:: readline() - Read one line, where "line" is a sequence of bytes ending with ``\n``. + Read one line, where "line" is a sequence of bytes + ending with ``\n``. - If EOF is received, and ``\n`` was not found, the method will - return the partial read bytes. + If EOF is received and ``\n`` was not found, the method + returns partially read data. - If the EOF was received and the internal buffer is empty, + If EOF is received and the internal buffer is empty, return an empty ``bytes`` object. - This method is a :ref:`coroutine `. - .. coroutinemethod:: readexactly(n) - Read exactly *n* bytes. Raise an :exc:`IncompleteReadError` if the end of - the stream is reached before *n* can be read, the - :attr:`IncompleteReadError.partial` attribute of the exception contains - the partial read bytes. + Read exactly *n* bytes. - This method is a :ref:`coroutine `. + Raise an :exc:`IncompleteReadError` if EOF is reached before *n* + can be read. Use the :attr:`IncompleteReadError.partial` + attribute to get the partially read data. .. coroutinemethod:: readuntil(separator=b'\\n') - Read data from the stream until ``separator`` is found. + Read data from the stream until *separator* is found. On success, the data and separator will be removed from the internal buffer (consumed). Returned data will include the separator at the end. - Configured stream limit is used to check result. Limit sets the - maximal length of data that can be returned, not counting the - separator. - - If an EOF occurs and the complete separator is still not found, - an :exc:`IncompleteReadError` exception will be - raised, and the internal buffer will be reset. The - :attr:`IncompleteReadError.partial` attribute may contain the - separator partially. - - If the data cannot be read because of over limit, a - :exc:`LimitOverrunError` exception will be raised, and the data - will be left in the internal buffer, so it can be read again. + If the amount of data read exceeds the configured stream limit, a + :exc:`LimitOverrunError` exception is raised, and the data + is left in the internal buffer and can be read again. + + If EOF is reached before the complete separator is found, + an :exc:`IncompleteReadError` exception is raised, and the internal + buffer is reset. The :attr:`IncompleteReadError.partial` attribute + may contain a portion of the separator. .. versionadded:: 3.5.2 .. method:: at_eof() - Return ``True`` if the buffer is empty and :meth:`feed_eof` was called. + Return ``True`` if the buffer is empty and :meth:`feed_eof` + was called. StreamWriter ============ -.. class:: StreamWriter(transport, protocol, reader, loop) +.. class:: StreamWriter - Wraps a Transport. + Represents a writer object that provides APIs to write data + to the IO stream. - This exposes :meth:`write`, :meth:`writelines`, :meth:`can_write_eof()`, - :meth:`write_eof`, :meth:`get_extra_info` and :meth:`close`. It adds - :meth:`drain` which returns an optional :class:`Future` on which you can - wait for flow control. It also adds a transport attribute which references - the :class:`Transport` directly. - - This class is :ref:`not thread safe `. - - .. attribute:: transport - - Transport. + It is not recommended to instantiate *StreamWriter* objects + directly; use :func:`open_connection` and :func:`start_server` + instead. .. method:: can_write_eof() - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. See :meth:`WriteTransport.can_write_eof`. - - .. method:: close() + Return *True* if the underlying transport supports + the :meth:`write_eof` method, *False* otherwise. - Close the transport: see :meth:`BaseTransport.close`. - - .. method:: is_closing() - - Return ``True`` if the writer is closing or is closed. - - .. versionadded:: 3.7 - - .. coroutinemethod:: wait_closed() - - Wait until the writer is closed. - - Should be called after :meth:`close` to wait until the underlying - connection (and the associated transport/protocol pair) is closed. - - .. versionadded:: 3.7 - - .. coroutinemethod:: drain() - - Let the write buffer of the underlying transport a chance to be flushed. - - The intended use is to write:: - - w.write(data) - await w.drain() + .. method:: write_eof() - When the size of the transport buffer reaches the high-water limit (the - protocol is paused), block until the size of the buffer is drained down - to the low-water limit and the protocol is resumed. When there is nothing - to wait for, the yield-from continues immediately. + Close the write end of the stream after the buffered write + data is flushed. - Yielding from :meth:`drain` gives the opportunity for the loop to - schedule the write operation and flush the buffer. It should especially - be used when a possibly large amount of data is written to the transport, - and the coroutine does not yield-from between calls to :meth:`write`. + .. attribute:: transport - This method is a :ref:`coroutine `. + Return the underlying asyncio transport. .. method:: get_extra_info(name, default=None) - Return optional transport information: see - :meth:`BaseTransport.get_extra_info`. + Access optional transport information; see + :meth:`BaseTransport.get_extra_info` for details. .. method:: write(data) - Write some *data* bytes to the transport: see - :meth:`WriteTransport.write`. - - .. method:: writelines(data) - - Write a list (or any iterable) of data bytes to the transport: - see :meth:`WriteTransport.writelines`. - - .. method:: write_eof() - - Close the write end of the transport after flushing buffered data: - see :meth:`WriteTransport.write_eof`. + Write *data* to the stream. + This method is not subject to flow control. Calls to ``write()`` should + be followed by :meth:`drain`. -StreamReaderProtocol -==================== - -.. class:: StreamReaderProtocol(stream_reader, client_connected_cb=None, loop=None) - - Trivial helper class to adapt between :class:`Protocol` and - :class:`StreamReader`. Subclass of :class:`Protocol`. - - *stream_reader* is a :class:`StreamReader` instance, *client_connected_cb* - is an optional function called with (stream_reader, stream_writer) when a - connection is made, *loop* is the event loop instance to use. + .. method:: writelines(data) - (This is a helper class instead of making :class:`StreamReader` itself a - :class:`Protocol` subclass, because the :class:`StreamReader` has other - potential uses, and to prevent the user of the :class:`StreamReader` from - accidentally calling inappropriate methods of the protocol.) + Write a list (or any iterable) of bytes to the stream. + This method is not subject to flow control. Calls to ``writelines()`` + should be followed by :meth:`drain`. -IncompleteReadError -=================== + .. coroutinemethod:: drain() -.. exception:: IncompleteReadError + Wait until it is appropriate to resume writing to the stream. + Example:: - Incomplete read error, subclass of :exc:`EOFError`. + writer.write(data) + await writer.drain() - .. attribute:: expected + This is a flow control method that interacts with the underlying + IO write buffer. When the size of the buffer reaches + the high watermark, *drain()* blocks until the size of the + buffer is drained down to the low watermark and writing can + be resumed. When there is nothing to wait for, the :meth:`drain` + returns immediately. - Total number of expected bytes (:class:`int`). + .. method:: close() - .. attribute:: partial + Close the stream. - Read bytes string before the end of stream was reached (:class:`bytes`). + .. method:: is_closing() + Return ``True`` if the stream is closed or in the process of + being closed. -LimitOverrunError -================= + .. versionadded:: 3.7 -.. exception:: LimitOverrunError + .. coroutinemethod:: wait_closed() - Reached the buffer limit while looking for a separator. + Wait until the stream is closed. - .. attribute:: consumed + Should be called after :meth:`close` to wait until the underlying + connection is closed. - Total number of to be consumed bytes. + .. versionadded:: 3.7 -Stream examples -=============== +Examples +======== .. _asyncio-tcp-echo-client-streams: @@ -354,28 +308,26 @@ import asyncio - async def tcp_echo_client(message, loop): - reader, writer = await asyncio.open_connection('127.0.0.1', 8888, - loop=loop) + async def tcp_echo_client(message): + reader, writer = await asyncio.open_connection( + '127.0.0.1', 8888) - print('Send: %r' % message) + print(f'Send: {message!r}') writer.write(message.encode()) data = await reader.read(100) - print('Received: %r' % data.decode()) + print(f'Received: {data.decode()!r}') - print('Close the socket') + print('Close the connection') writer.close() - message = 'Hello World!' - loop = asyncio.get_event_loop() - loop.run_until_complete(tcp_echo_client(message, loop)) - loop.close() + asyncio.run(tcp_echo_client('Hello World!')) + .. seealso:: - The :ref:`TCP echo client protocol ` - example uses the :meth:`AbstractEventLoop.create_connection` method. + The :ref:`TCP echo client protocol ` + example uses the low-level :meth:`loop.create_connection` method. .. _asyncio-tcp-echo-server-streams: @@ -391,35 +343,33 @@ data = await reader.read(100) message = data.decode() addr = writer.get_extra_info('peername') - print("Received %r from %r" % (message, addr)) - print("Send: %r" % message) + print(f"Received {message!r} from {addr!r}") + + print(f"Send: {message!r}") writer.write(data) await writer.drain() - print("Close the client socket") + print("Close the connection") writer.close() - loop = asyncio.get_event_loop() - coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888, loop=loop) - server = loop.run_until_complete(coro) - - # Serve requests until Ctrl+C is pressed - print('Serving on {}'.format(server.sockets[0].getsockname())) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - # Close the server - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() + async def main(): + server = await asyncio.start_server( + handle_echo, '127.0.0.1', 8888) + + addr = server.sockets[0].getsockname() + print(f'Serving on {addr}') + + async with server: + await server.serve_forever() + + asyncio.run(main()) + .. seealso:: - The :ref:`TCP echo server protocol ` - example uses the :meth:`AbstractEventLoop.create_server` method. + The :ref:`TCP echo server protocol ` + example uses the :meth:`loop.create_server` method. Get HTTP headers @@ -431,34 +381,37 @@ import urllib.parse import sys - @asyncio.coroutine - def print_http_headers(url): + async def print_http_headers(url): url = urllib.parse.urlsplit(url) if url.scheme == 'https': - connect = asyncio.open_connection(url.hostname, 443, ssl=True) + reader, writer = await asyncio.open_connection( + url.hostname, 443, ssl=True) else: - connect = asyncio.open_connection(url.hostname, 80) - reader, writer = await connect - query = ('HEAD {path} HTTP/1.0\r\n' - 'Host: {hostname}\r\n' - '\r\n').format(path=url.path or '/', hostname=url.hostname) + reader, writer = await asyncio.open_connection( + url.hostname, 80) + + query = ( + f"HEAD {url.path or '/'} HTTP/1.0\r\n" + f"Host: {url.hostname}\r\n" + f"\r\n" + ) + writer.write(query.encode('latin-1')) while True: line = await reader.readline() if not line: break + line = line.decode('latin1').rstrip() if line: - print('HTTP header> %s' % line) + print(f'HTTP header> {line}') # Ignore the body, close the socket writer.close() url = sys.argv[1] - loop = asyncio.get_event_loop() - task = asyncio.ensure_future(print_http_headers(url)) - loop.run_until_complete(task) - loop.close() + asyncio.run(print_http_headers(url)) + Usage:: @@ -468,7 +421,8 @@ python example.py https://example.com/path/page.html -.. _asyncio-register-socket-streams: + +.. _asyncio_example_create_connection-streams: Register an open socket to wait for data using streams ------------------------------------------------------ @@ -477,14 +431,18 @@ :func:`open_connection` function:: import asyncio - from socket import socketpair + import socket + + async def wait_for_data(): + # Get a reference to the current event loop because + # we want to access low-level APIs. + loop = asyncio.get_running_loop() - async def wait_for_data(loop): - # Create a pair of connected sockets - rsock, wsock = socketpair() + # Create a pair of connected sockets. + rsock, wsock = socket.socketpair() - # Register the open socket to wait for data - reader, writer = await asyncio.open_connection(sock=rsock, loop=loop) + # Register the open socket to wait for data. + reader, writer = await asyncio.open_connection(sock=rsock) # Simulate the reception of data from the network loop.call_soon(wsock.send, 'abc'.encode()) @@ -499,17 +457,14 @@ # Close the second socket wsock.close() - loop = asyncio.get_event_loop() - loop.run_until_complete(wait_for_data(loop)) - loop.close() + asyncio.run(wait_for_data()) .. seealso:: The :ref:`register an open socket to wait for data using a protocol - ` example uses a low-level protocol created by the - :meth:`AbstractEventLoop.create_connection` method. + ` example uses a low-level protocol and + the :meth:`loop.create_connection` method. The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`AbstractEventLoop.add_reader` method to register the file descriptor of a - socket. + ` example uses the low-level + :meth:`loop.add_reader` method to watch a file descriptor. diff -Nru python3.7-3.7.0/Doc/library/asyncio-subprocess.rst python3.7-3.7.1/Doc/library/asyncio-subprocess.rst --- python3.7-3.7.0/Doc/library/asyncio-subprocess.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-subprocess.rst 2018-10-20 06:04:19.000000000 +0000 @@ -2,245 +2,220 @@ .. _asyncio-subprocess: -Subprocess -========== +============ +Subprocesses +============ -**Source code:** :source:`Lib/asyncio/subprocess.py` +This section describes high-level async/await asyncio APIs to +create and manage subprocesses. -Windows event loop ------------------- +.. _asyncio_example_subprocess_shell: -On Windows, the default event loop is :class:`SelectorEventLoop` which does not -support subprocesses. :class:`ProactorEventLoop` should be used instead. -Example to use it on Windows:: +Here's an example of how asyncio can run a shell command and +obtain its result:: - import asyncio, sys - - if sys.platform == 'win32': - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - -.. seealso:: - - :ref:`Available event loops ` and :ref:`Platform - support `. - - -Create a subprocess: high-level API using Process -------------------------------------------------- - -.. coroutinefunction:: create_subprocess_exec(\*args, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) - - Create a subprocess. - - The *limit* parameter sets the buffer limit passed to the - :class:`StreamReader`. See :meth:`AbstractEventLoop.subprocess_exec` for other - parameters. - - Return a :class:`~asyncio.subprocess.Process` instance. - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) - - Run the shell command *cmd*. - - The *limit* parameter sets the buffer limit passed to the - :class:`StreamReader`. See :meth:`AbstractEventLoop.subprocess_shell` for other - parameters. - - Return a :class:`~asyncio.subprocess.Process` instance. - - It is the application's responsibility to ensure that all whitespace and - metacharacters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and shell metacharacters in strings that are going to be - used to construct shell commands. + import asyncio - This function is a :ref:`coroutine `. + async def run(cmd): + proc = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) -Use the :meth:`AbstractEventLoop.connect_read_pipe` and -:meth:`AbstractEventLoop.connect_write_pipe` methods to connect pipes. + stdout, stderr = await proc.communicate() + print(f'[{cmd!r} exited with {proc.returncode}]') + if stdout: + print(f'[stdout]\n{stdout.decode()}') + if stderr: + print(f'[stderr]\n{stderr.decode()}') -Create a subprocess: low-level API using subprocess.Popen ---------------------------------------------------------- + asyncio.run(run('ls /zzz')) -Run subprocesses asynchronously using the :mod:`subprocess` module. +will print:: -.. coroutinemethod:: AbstractEventLoop.subprocess_exec(protocol_factory, \*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) + ['ls /zzz' exited with 1] + [stderr] + ls: /zzz: No such file or directory - Create a subprocess from one or more string arguments (character strings or - bytes strings encoded to the :ref:`filesystem encoding - `), where the first string - specifies the program to execute, and the remaining strings specify the - program's arguments. (Thus, together the string arguments form the - ``sys.argv`` value of the program, assuming it is a Python script.) This is - similar to the standard library :class:`subprocess.Popen` class called with - shell=False and the list of strings passed as the first argument; - however, where :class:`~subprocess.Popen` takes a single argument which is - list of strings, :func:`subprocess_exec` takes multiple string arguments. +Because all asyncio subprocess functions are asynchronous and asyncio +provides many tools to work with such functions, it is easy to execute +and monitor multiple subprocesses in parallel. It is indeed trivial +to modify the above example to run several commands simultaneously:: - The *protocol_factory* must instantiate a subclass of the - :class:`asyncio.SubprocessProtocol` class. + async def main(): + await asyncio.gather( + run('ls /zzz'), + run('sleep 1; echo "hello"')) - Other parameters: + asyncio.run(main()) - * *stdin*: Either a file-like object representing the pipe to be connected - to the subprocess's standard input stream using - :meth:`~AbstractEventLoop.connect_write_pipe`, or the constant - :const:`subprocess.PIPE` (the default). By default a new pipe will be - created and connected. +See also the `Examples`_ subsection. - * *stdout*: Either a file-like object representing the pipe to be connected - to the subprocess's standard output stream using - :meth:`~AbstractEventLoop.connect_read_pipe`, or the constant - :const:`subprocess.PIPE` (the default). By default a new pipe will be - created and connected. - * *stderr*: Either a file-like object representing the pipe to be connected - to the subprocess's standard error stream using - :meth:`~AbstractEventLoop.connect_read_pipe`, or one of the constants - :const:`subprocess.PIPE` (the default) or :const:`subprocess.STDOUT`. - By default a new pipe will be created and connected. When - :const:`subprocess.STDOUT` is specified, the subprocess's standard error - stream will be connected to the same pipe as the standard output stream. +Creating Subprocesses +===================== - * All other keyword arguments are passed to :class:`subprocess.Popen` - without interpretation, except for *bufsize*, *universal_newlines* and - *shell*, which should not be specified at all. +.. coroutinefunction:: create_subprocess_exec(\*args, stdin=None, \ + stdout=None, stderr=None, loop=None, \ + limit=None, \*\*kwds) - Returns a pair of ``(transport, protocol)``, where *transport* is an - instance of :class:`BaseSubprocessTransport`. + Create a subprocess. - This method is a :ref:`coroutine `. + The *limit* argument sets the buffer limit for :class:`StreamReader` + wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` + (if :attr:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). - See the constructor of the :class:`subprocess.Popen` class for parameters. + Return a :class:`~asyncio.subprocess.Process` instance. -.. coroutinemethod:: AbstractEventLoop.subprocess_shell(protocol_factory, cmd, \*, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) + See the documentation of :meth:`loop.subprocess_exec` for other + parameters. - Create a subprocess from *cmd*, which is a character string or a bytes - string encoded to the :ref:`filesystem encoding `, - using the platform's "shell" syntax. This is similar to the standard library - :class:`subprocess.Popen` class called with ``shell=True``. +.. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \ + stdout=None, stderr=None, loop=None, \ + limit=None, \*\*kwds) + + Run the *cmd* shell command. + + The *limit* argument sets the buffer limit for :class:`StreamReader` + wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` + (if :attr:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). - The *protocol_factory* must instantiate a subclass of the - :class:`asyncio.SubprocessProtocol` class. + Return a :class:`~asyncio.subprocess.Process` instance. - See :meth:`~AbstractEventLoop.subprocess_exec` for more details about - the remaining arguments. + See the documentation of :meth:`loop.subprocess_shell` for other + parameters. - Returns a pair of ``(transport, protocol)``, where *transport* is an - instance of :class:`BaseSubprocessTransport`. +.. important:: It is the application's responsibility to ensure that all whitespace and - metacharacters are quoted appropriately to avoid `shell injection + special characters are quoted appropriately to avoid `shell injection `_ vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and shell metacharacters in strings that are going to be - used to construct shell commands. + escape whitespace and special shell characters in strings that are going + to be used to construct shell commands. - This method is a :ref:`coroutine `. +.. note:: + + The default asyncio event loop implementation on **Windows** does not + support subprocesses. Subprocesses are available for Windows if a + :class:`ProactorEventLoop` is used. + See :ref:`Subprocess Support on Windows ` + for details. .. seealso:: - The :meth:`AbstractEventLoop.connect_read_pipe` and - :meth:`AbstractEventLoop.connect_write_pipe` methods. + asyncio also has the following *low-level* APIs to work with subprocesses: + :meth:`loop.subprocess_exec`, :meth:`loop.subprocess_shell`, + :meth:`loop.connect_read_pipe`, :meth:`loop.connect_write_pipe`, + as well as the :ref:`Subprocess Transports ` + and :ref:`Subprocess Protocols `. Constants ---------- +========= .. data:: asyncio.subprocess.PIPE - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that a pipe to the standard stream should be opened. + Can be passed to the *stdin*, *stdout* or *stderr* parameters. + + If *PIPE* is passed to *stdin* argument, the + :attr:`Process.stdin ` attribute + will point to a :class:`StreamWriter` instance. + + If *PIPE* is passed to *stdout* or *stderr* arguments, the + :attr:`Process.stdout ` and + :attr:`Process.stderr ` + attributes will point to :class:`StreamReader` instances. .. data:: asyncio.subprocess.STDOUT - Special value that can be used as the *stderr* argument to - :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that standard error should go into the same handle as standard - output. + Special value that can be used as the *stderr* argument and indicates + that standard error should be redirected into standard output. .. data:: asyncio.subprocess.DEVNULL Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that the special file :data:`os.devnull` will be used. + to process creation functions. It indicates that the special file + :data:`os.devnull` will be used for the corresponding subprocess stream. + +Interacting with Subprocesses +============================= -Process -------- +Both :func:`create_subprocess_exec` and :func:`create_subprocess_shell` +functions return instances of the *Process* class. *Process* is a high-level +wrapper that allows communicating with subprocesses and watching for +their completion. .. class:: asyncio.subprocess.Process - A subprocess created by the :func:`create_subprocess_exec` or the - :func:`create_subprocess_shell` function. + An object that wraps OS processes created by the + :func:`create_subprocess_exec` and :func:`create_subprocess_shell` + functions. - The API of the :class:`~asyncio.subprocess.Process` class was designed to be - close to the API of the :class:`subprocess.Popen` class, but there are some - differences: - - * There is no explicit :meth:`~subprocess.Popen.poll` method - * The :meth:`~subprocess.Popen.communicate` and - :meth:`~subprocess.Popen.wait` methods don't take a *timeout* parameter: - use the :func:`wait_for` function - * The *universal_newlines* parameter is not supported (only bytes strings - are supported) - * The :meth:`~asyncio.subprocess.Process.wait` method of - the :class:`~asyncio.subprocess.Process` class is asynchronous whereas the - :meth:`~subprocess.Popen.wait` method of the :class:`~subprocess.Popen` - class is implemented as a busy loop. + This class is designed to have a similar API to the + :class:`subprocess.Popen` class, but there are some + notable differences: - This class is :ref:`not thread safe `. See also the - :ref:`Subprocess and threads ` section. + * unlike Popen, Process instances do not have an equivalent to + the :meth:`~subprocess.Popen.poll` method; - .. coroutinemethod:: wait() + * the :meth:`~asyncio.subprocess.Process.communicate` and + :meth:`~asyncio.subprocess.Process.wait` methods don't have a + *timeout* parameter: use the :func:`wait_for` function; - Wait for child process to terminate. Set and return :attr:`returncode` - attribute. + * the :meth:`Process.wait() ` method + is asynchronous, whereas :meth:`subprocess.Popen.wait` method + is implemented as a blocking busy loop; - This method is a :ref:`coroutine `. + * the *universal_newlines* parameter is not supported. - .. note:: + This class is :ref:`not thread safe `. - This will deadlock when using ``stdout=PIPE`` or ``stderr=PIPE`` and - the child process generates enough output to a pipe such that it - blocks waiting for the OS pipe buffer to accept more data. Use the - :meth:`communicate` method when using pipes to avoid that. + See also the :ref:`Subprocess and Threads ` + section. - .. coroutinemethod:: communicate(input=None) + .. coroutinemethod:: wait() - Interact with process: Send data to stdin. Read data from stdout and - stderr, until end-of-file is reached. Wait for process to terminate. - The optional *input* argument should be data to be sent to the child - process, or ``None``, if no data should be sent to the child. The type - of *input* must be bytes. - - :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. - - If a :exc:`BrokenPipeError` or :exc:`ConnectionResetError` exception is - raised when writing *input* into stdin, the exception is ignored. It - occurs when the process exits before all data are written into stdin. - - Note that if you want to send data to the process's stdin, you need to - create the Process object with ``stdin=PIPE``. Similarly, to get anything - other than ``None`` in the result tuple, you need to give ``stdout=PIPE`` - and/or ``stderr=PIPE`` too. + Wait for the child process to terminate. - This method is a :ref:`coroutine `. + Set and return the :attr:`returncode` attribute. .. note:: - The data read is buffered in memory, so do not use this method if the - data size is large or unlimited. + This method can deadlock when using ``stdout=PIPE`` or + ``stderr=PIPE`` and the child process generates so much output + that it blocks waiting for the OS pipe buffer to accept + more data. Use the :meth:`communicate` method when using pipes + to avoid this condition. + + .. coroutinemethod:: communicate(input=None) + + Interact with process: + + 1. send data to *stdin* (if *input* is not ``None``); + 2. read data from *stdout* and *stderr*, until EOF is reached; + 3. wait for process to terminate. + + The optional *input* argument is the data (:class:`bytes` object) + that will be sent to the child process. + + Return a tuple ``(stdout_data, stderr_data)``. + + If either :exc:`BrokenPipeError` or :exc:`ConnectionResetError` + exception is raised when writing *input* into *stdin*, the + exception is ignored. This condition occurs when the process + exits before all data are written into *stdin*. + + If it is desired to send data to the process' *stdin*, + the process needs to be created with ``stdin=PIPE``. Similarly, + to get anything other than ``None`` in the result tuple, the + process has to be created with ``stdout=PIPE`` and/or + ``stderr=PIPE`` arguments. - .. versionchanged:: 3.4.2 - The method now ignores :exc:`BrokenPipeError` and - :exc:`ConnectionResetError`. + Note, that the data read is buffered in memory, so do not use + this method if the data size is large or unlimited. .. method:: send_signal(signal) @@ -255,67 +230,81 @@ .. method:: terminate() - Stop the child. On Posix OSs the method sends :py:data:`signal.SIGTERM` - to the child. On Windows the Win32 API function - :c:func:`TerminateProcess` is called to stop the child. + Stop the child process. + + On POSIX systems this method sends :py:data:`signal.SIGTERM` to the + child process. + + On Windows the Win32 API function :c:func:`TerminateProcess` is + called to stop the child process. .. method:: kill() - Kills the child. On Posix OSs the function sends :py:data:`SIGKILL` to - the child. On Windows :meth:`kill` is an alias for :meth:`terminate`. + Kill the child. + + On POSIX systems this method sends :py:data:`SIGKILL` to the child + process. + + On Windows this method is an alias for :meth:`terminate`. .. attribute:: stdin - Standard input stream (:class:`StreamWriter`), ``None`` if the process - was created with ``stdin=None``. + Standard input stream (:class:`StreamWriter`) or ``None`` + if the process was created with ``stdin=None``. .. attribute:: stdout - Standard output stream (:class:`StreamReader`), ``None`` if the process - was created with ``stdout=None``. + Standard output stream (:class:`StreamReader`) or ``None`` + if the process was created with ``stdout=None``. .. attribute:: stderr - Standard error stream (:class:`StreamReader`), ``None`` if the process - was created with ``stderr=None``. + Standard error stream (:class:`StreamReader`) or ``None`` + if the process was created with ``stderr=None``. .. warning:: - Use the :meth:`communicate` method rather than :attr:`.stdin.write - `, :attr:`.stdout.read ` or :attr:`.stderr.read ` - to avoid deadlocks due to streams pausing reading or writing and blocking - the child process. + Use the :meth:`communicate` method rather than + :attr:`process.stdin.write() `, + :attr:`await process.stdout.read() ` or + :attr:`await process.stderr.read `. + This avoids deadlocks due to streams pausing reading or writing + and blocking the child process. .. attribute:: pid - The identifier of the process. + Process identification number (PID). Note that for processes created by the :func:`create_subprocess_shell` - function, this attribute is the process identifier of the spawned shell. + function, this attribute is the PID of the spawned shell. .. attribute:: returncode - Return code of the process when it exited. A ``None`` value indicates - that the process has not terminated yet. + Return code of the process when it exits. - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (Unix only). + A ``None`` value indicates that the process has not terminated yet. + + A negative value ``-N`` indicates that the child was terminated + by signal ``N`` (POSIX only). .. _asyncio-subprocess-threads: -Subprocess and threads +Subprocess and Threads ---------------------- -asyncio supports running subprocesses from different threads, but there -are limits: +Standard asyncio event loop supports running subprocesses from +different threads, but there are limitations: + +* An event loop must run in the main thread. -* An event loop must run in the main thread -* The child watcher must be instantiated in the main thread, before executing - subprocesses from other threads. Call the :func:`get_child_watcher` - function in the main thread to instantiate the child watcher. +* The child watcher must be instantiated in the main thread + before executing subprocesses from other threads. Call the + :func:`get_child_watcher` function in the main thread to instantiate + the child watcher. -The :class:`asyncio.subprocess.Process` class is not thread safe. +Note that alternative event loop implementations might not share +the above limitations; please refer to their documentation. .. seealso:: @@ -323,98 +312,45 @@ ` section. -Subprocess examples -------------------- +Examples +-------- -Subprocess using transport and protocol -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +An example using the :class:`~asyncio.subprocess.Process` class to +control a subprocess and the :class:`StreamReader` class to read from +its standard output. -Example of a subprocess protocol using to get the output of a subprocess and to -wait for the subprocess exit. The subprocess is created by the -:meth:`AbstractEventLoop.subprocess_exec` method:: +.. _asyncio_example_create_subprocess_exec: - import asyncio - import sys - - class DateProtocol(asyncio.SubprocessProtocol): - def __init__(self, exit_future): - self.exit_future = exit_future - self.output = bytearray() - - def pipe_data_received(self, fd, data): - self.output.extend(data) - - def process_exited(self): - self.exit_future.set_result(True) - - async def get_date(loop): - code = 'import datetime; print(datetime.datetime.now())' - exit_future = asyncio.Future(loop=loop) - - # Create the subprocess controlled by the protocol DateProtocol, - # redirect the standard output into a pipe - transport, protocol = await loop.subprocess_exec( - lambda: DateProtocol(exit_future), - sys.executable, '-c', code, - stdin=None, stderr=None) - - # Wait for the subprocess exit using the process_exited() method - # of the protocol - await exit_future - - # Close the stdout pipe - transport.close() - - # Read the output which was collected by the pipe_data_received() - # method of the protocol - data = bytes(protocol.output) - return data.decode('ascii').rstrip() - - if sys.platform == "win32": - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - else: - loop = asyncio.get_event_loop() - - date = loop.run_until_complete(get_date(loop)) - print("Current date: %s" % date) - loop.close() - - -Subprocess using streams -^^^^^^^^^^^^^^^^^^^^^^^^ - -Example using the :class:`~asyncio.subprocess.Process` class to control the -subprocess and the :class:`StreamReader` class to read from the standard -output. The subprocess is created by the :func:`create_subprocess_exec` +The subprocess is created by the :func:`create_subprocess_exec` function:: - import asyncio.subprocess + import asyncio import sys - @asyncio.coroutine - def get_date(): + async def get_date(): code = 'import datetime; print(datetime.datetime.now())' - # Create the subprocess, redirect the standard output into a pipe + # Create the subprocess; redirect the standard output + # into a pipe. proc = await asyncio.create_subprocess_exec( sys.executable, '-c', code, stdout=asyncio.subprocess.PIPE) - # Read one line of output + # Read one line of output. data = await proc.stdout.readline() line = data.decode('ascii').rstrip() - # Wait for the subprocess exit + # Wait for the subprocess exit. await proc.wait() return line if sys.platform == "win32": - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - else: - loop = asyncio.get_event_loop() - - date = loop.run_until_complete(get_date()) - print("Current date: %s" % date) - loop.close() + asyncio.set_event_loop_policy( + asyncio.WindowsProactorEventLoopPolicy()) + + date = asyncio.run(get_date()) + print(f"Current date: {date}") + + +See also the :ref:`same example ` +written using low-level APIs. diff -Nru python3.7-3.7.0/Doc/library/asyncio-sync.rst python3.7-3.7.1/Doc/library/asyncio-sync.rst --- python3.7-3.7.0/Doc/library/asyncio-sync.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-sync.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,172 +1,206 @@ .. currentmodule:: asyncio + .. _asyncio-sync: -Synchronization primitives +========================== +Synchronization Primitives ========================== -**Source code:** :source:`Lib/asyncio/locks.py` +asyncio synchronization primitives are designed to be similar to +those of the :mod:`threading` module with two important caveats: -Locks: +* asyncio primitives are not thread-safe, therefore they should not + be used for OS thread synchronization (use :mod:`threading` for + that); + +* methods of these synchronization primitives do not accept the *timeout* + argument; use the :func:`asyncio.wait_for` function to perform + operations with timeouts. + +asyncio has the following basic sychronization primitives: * :class:`Lock` * :class:`Event` * :class:`Condition` - -Semaphores: - * :class:`Semaphore` * :class:`BoundedSemaphore` -asyncio lock API was designed to be close to classes of the :mod:`threading` -module (:class:`~threading.Lock`, :class:`~threading.Event`, -:class:`~threading.Condition`, :class:`~threading.Semaphore`, -:class:`~threading.BoundedSemaphore`), but it has no *timeout* parameter. The -:func:`asyncio.wait_for` function can be used to cancel a task after a timeout. + +--------- Lock ----- +==== .. class:: Lock(\*, loop=None) - Primitive lock objects. + Implements a mutex lock for asyncio tasks. Not thread-safe. - A primitive lock is a synchronization primitive that is not owned by a - particular coroutine when locked. A primitive lock is in one of two states, - 'locked' or 'unlocked'. - - The lock is created in the unlocked state. - It has two basic methods, :meth:`acquire` and :meth:`release`. - When the state is unlocked, acquire() changes the state to - locked and returns immediately. When the state is locked, acquire() blocks - until a call to release() in another coroutine changes it to unlocked, then - the acquire() call resets it to locked and returns. The release() method - should only be called in the locked state; it changes the state to unlocked - and returns immediately. If an attempt is made to release an unlocked lock, - a :exc:`RuntimeError` will be raised. - - When more than one coroutine is blocked in acquire() waiting for the state - to turn to unlocked, only one coroutine proceeds when a release() call - resets the state to unlocked; first coroutine which is blocked in acquire() - is being processed. + An asyncio lock can be used to guarantee exclusive access to a + shared resource. - :meth:`acquire` is a coroutine and should be called with ``await``. + The preferred way to use a Lock is an :keyword:`async with` + statement:: - Locks support the :ref:`context management protocol `. + lock = asyncio.Lock() - This class is :ref:`not thread safe `. + # ... later + async with lock: + # access shared state - .. method:: locked() + which is equivalent to:: - Return ``True`` if the lock is acquired. + lock = asyncio.Lock() - .. coroutinemethod:: acquire() + # ... later + await lock.acquire() + try: + # access shared state + finally: + lock.release() - Acquire a lock. + .. coroutinemethod:: acquire() - This method blocks until the lock is unlocked, then sets it to locked and - returns ``True``. + Acquire the lock. - This method is a :ref:`coroutine `. + This method waits until the lock is *unlocked*, sets it to + *locked* and returns ``True``. .. method:: release() - Release a lock. + Release the lock. - When the lock is locked, reset it to unlocked, and return. If any other - coroutines are blocked waiting for the lock to become unlocked, allow - exactly one of them to proceed. + When the lock is *locked*, reset it to *unlocked* and return. - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. + If the lock is *unlocked*, a :exc:`RuntimeError` is raised. - There is no return value. + .. method:: locked() + + Return ``True`` if the lock is *locked*. Event ------ +===== .. class:: Event(\*, loop=None) - An Event implementation, asynchronous equivalent to :class:`threading.Event`. + An event object. Not thread-safe. - Class implementing event objects. An event manages a flag that can be set to - true with the :meth:`set` method and reset to false with the :meth:`clear` - method. The :meth:`wait` method blocks until the flag is true. The flag is - initially false. + An asyncio event can be used to notify multiple asyncio tasks + that some event has happened. - This class is :ref:`not thread safe `. + An Event object manages an internal flag that can be set to *true* + with the :meth:`set` method and reset to *false* with the + :meth:`clear` method. The :meth:`wait` method blocks until the + flag is set to *true*. The flag is set to *false* initially. - .. method:: clear() + .. _asyncio_example_sync_event: - Reset the internal flag to false. Subsequently, coroutines calling - :meth:`wait` will block until :meth:`set` is called to set the internal - flag to true again. + Example:: - .. method:: is_set() + async def waiter(event): + print('waiting for it ...') + await event.wait() + print('... got it!') - Return ``True`` if and only if the internal flag is true. + async def main(): + # Create an Event object. + event = asyncio.Event() - .. method:: set() + # Spawn a Task to wait until 'event' is set. + waiter_task = asyncio.create_task(waiter(event)) + + # Sleep for 1 second and set the event. + await asyncio.sleep(1) + event.set() - Set the internal flag to true. All coroutines waiting for it to become - true are awakened. Coroutine that call :meth:`wait` once the flag is true - will not block at all. + # Wait until the waiter task is finished. + await waiter_task + + asyncio.run(main()) .. coroutinemethod:: wait() - Block until the internal flag is true. + Wait until the event is set. + + If the event is set, return ``True`` immediately. + Otherwise block until another task calls :meth:`set`. - If the internal flag is true on entry, return ``True`` immediately. - Otherwise, block until another coroutine calls :meth:`set` to set the - flag to true, then return ``True``. + .. method:: set() + + Set the event. - This method is a :ref:`coroutine `. + All tasks waiting for event to be set will be immediately + awakened. + + .. method:: clear() + + Clear (unset) the event. + + Tasks awaiting on :meth:`wait` will now block until the + :meth:`set` method is called again. + + .. method:: is_set() + + Return ``True`` if the event is set. Condition ---------- +========= .. class:: Condition(lock=None, \*, loop=None) - A Condition implementation, asynchronous equivalent to - :class:`threading.Condition`. + A Condition object. Not thread-safe. - This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. + An asyncio condition primitive can be used by a task to wait for + some event to happen and then get exclusive access to a shared + resource. - If the *lock* argument is given and not ``None``, it must be a :class:`Lock` - object, and it is used as the underlying lock. Otherwise, - a new :class:`Lock` object is created and used as the underlying lock. + In essence, a Condition object combines the functionality + of an :class:`Event` and a :class:`Lock`. It is possible to have + multiple Condition objects share one Lock, which allows coordinating + exclusive access to a shared resource between different tasks + interested in particular states of that shared resource. - Conditions support the :ref:`context management protocol - `. + The optional *lock* argument must be a :class:`Lock` object or + ``None``. In the latter case a new Lock object is created + automatically. - This class is :ref:`not thread safe `. + The preferred way to use a Condition is an :keyword:`async with` + statement:: - .. coroutinemethod:: acquire() + cond = asyncio.Condition() - Acquire the underlying lock. + # ... later + async with cond: + await cond.wait() - This method blocks until the lock is unlocked, then sets it to locked and - returns ``True``. + which is equivalent to:: - This method is a :ref:`coroutine `. + cond = asyncio.Condition() - .. method:: notify(n=1) + # ... later + await lock.acquire() + try: + await cond.wait() + finally: + lock.release() - By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. + .. coroutinemethod:: acquire() + + Acquire the underlying lock. - This method wakes up at most *n* of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. + This method waits until the underlying lock is *unlocked*, + sets it to *locked* and returns ``True``. - .. note:: + .. method:: notify(n=1) - An awakened coroutine does not actually return from its :meth:`wait` - call until it can reacquire the lock. Since :meth:`notify` does not - release the lock, its caller should. + Wake up at most *n* tasks (1 by default) waiting on this + condition. The method is no-op if no tasks are waiting. + + The lock must be acquired before this method is called and + released shortly after. If called with an *unlocked* lock + a :exc:`RuntimeError` error is raised. .. method:: locked() @@ -174,78 +208,87 @@ .. method:: notify_all() - Wake up all coroutines waiting on this condition. This method acts like - :meth:`notify`, but wakes up all waiting coroutines instead of one. If the - calling coroutine has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. + Wake up all tasks waiting on this condition. - .. method:: release() + This method acts like :meth:`notify`, but wakes up all waiting + tasks. - Release the underlying lock. + The lock must be acquired before this method is called and + released shortly after. If called with an *unlocked* lock + a :exc:`RuntimeError` error is raised. - When the lock is locked, reset it to unlocked, and return. If any other - coroutines are blocked waiting for the lock to become unlocked, allow - exactly one of them to proceed. + .. method:: release() - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. + Release the underlying lock. - There is no return value. + When invoked on an unlocked lock, a :exc:`RuntimeError` is + raised. .. coroutinemethod:: wait() Wait until notified. - If the calling coroutine has not acquired the lock when this method is + If the calling task has not acquired the lock when this method is called, a :exc:`RuntimeError` is raised. - This method releases the underlying lock, and then blocks until it is - awakened by a :meth:`notify` or :meth:`notify_all` call for the same - condition variable in another coroutine. Once awakened, it re-acquires - the lock and returns ``True``. - - This method is a :ref:`coroutine `. + This method releases the underlying lock, and then blocks until + it is awakened by a :meth:`notify` or :meth:`notify_all` call. + Once awakened, the Condition re-acquires its lock and this method + returns ``True``. .. coroutinemethod:: wait_for(predicate) - Wait until a predicate becomes true. + Wait until a predicate becomes *true*. - The predicate should be a callable which result will be interpreted as a - boolean value. The final predicate value is the return value. - - This method is a :ref:`coroutine `. + The predicate must be a callable which result will be + interpreted as a boolean value. The final value is the + return value. Semaphore ---------- +========= .. class:: Semaphore(value=1, \*, loop=None) - A Semaphore implementation. + A Semaphore object. Not thread-safe. A semaphore manages an internal counter which is decremented by each - :meth:`acquire` call and incremented by each :meth:`release` call. The - counter can never go below zero; when :meth:`acquire` finds that it is zero, - it blocks, waiting until some other coroutine calls :meth:`release`. - - The optional argument gives the initial value for the internal counter; it - defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` - is raised. - - Semaphores support the :ref:`context management protocol - `. - - This class is :ref:`not thread safe `. + :meth:`acquire` call and incremented by each :meth:`release` call. + The counter can never go below zero; when :meth:`acquire` finds + that it is zero, it blocks, waiting until some task calls + :meth:`release`. + + The optional *value* argument gives the initial value for the + internal counter (``1`` by default). If the given value is + less than ``0`` a :exc:`ValueError` is raised. + + The preferred way to use a Semaphore is an :keyword:`async with` + statement:: + + sem = asyncio.Semaphore(10) + + # ... later + async with sem: + # work with shared resource + + which is equivalent to:: + + sem = asyncio.Semaphore(10) + + # ... later + await sem.acquire() + try: + # work with shared resource + finally: + sem.release() .. coroutinemethod:: acquire() Acquire a semaphore. - If the internal counter is larger than zero on entry, decrement it by one - and return ``True`` immediately. If it is zero on entry, block, waiting - until some other coroutine has called :meth:`release` to make it larger - than ``0``, and then return ``True``. - - This method is a :ref:`coroutine `. + If the internal counter is greater than zero, decrement + it by one and return ``True`` immediately. If it is zero, wait + until a :meth:`release` is called and return ``True``. .. method:: locked() @@ -253,53 +296,30 @@ .. method:: release() - Release a semaphore, incrementing the internal counter by one. When it - was zero on entry and another coroutine is waiting for it to become - larger than zero again, wake up that coroutine. + Release a semaphore, incrementing the internal counter by one. + Can wake up a task waiting to acquire the semaphore. + + Unlike :class:`BoundedSemaphore`, :class:`Semaphore` allows + making more ``release()`` calls than ``acquire()`` calls. BoundedSemaphore ----------------- +================ .. class:: BoundedSemaphore(value=1, \*, loop=None) - A bounded semaphore implementation. Inherit from :class:`Semaphore`. - - This raises :exc:`ValueError` in :meth:`~Semaphore.release` if it would - increase the value above the initial value. + A bounded semaphore object. Not thread-safe. - Bounded semapthores support the :ref:`context management - protocol `. + Bounded Semaphore is a version of :class:`Semaphore` that raises + a :exc:`ValueError` in :meth:`~Semaphore.release` if it + increases the internal counter above the initial *value*. - This class is :ref:`not thread safe `. +--------- -.. _async-with-locks: - -Using locks, conditions and semaphores in the :keyword:`async with` statement ------------------------------------------------------------------------------ - -:class:`Lock`, :class:`Condition`, :class:`Semaphore`, and -:class:`BoundedSemaphore` objects can be used in :keyword:`async with` -statements. - -The :meth:`acquire` method will be called when the block is entered, -and :meth:`release` will be called when the block is exited. Hence, -the following snippet:: - - async with lock: - # do something... - -is equivalent to:: - - await lock.acquire() - try: - # do something... - finally: - lock.release() .. deprecated:: 3.7 - Lock acquiring using ``await lock`` or ``yield from lock`` and + Acquiring a lock using ``await lock`` or ``yield from lock`` and/or :keyword:`with` statement (``with await lock``, ``with (yield from - lock)``) are deprecated. + lock)``) is deprecated. Use ``async with lock`` instead. diff -Nru python3.7-3.7.0/Doc/library/asyncio-task.rst python3.7-3.7.1/Doc/library/asyncio-task.rst --- python3.7-3.7.0/Doc/library/asyncio-task.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/asyncio-task.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,562 +1,622 @@ .. currentmodule:: asyncio -Tasks and coroutines + +==================== +Coroutines and Tasks ==================== -**Source code:** :source:`Lib/asyncio/tasks.py` +This section outlines high-level asyncio APIs to work with coroutines +and Tasks. + +.. contents:: + :depth: 1 + :local: -**Source code:** :source:`Lib/asyncio/coroutines.py` .. _coroutine: Coroutines ----------- +========== -Coroutines used with :mod:`asyncio` may be implemented using the -:keyword:`async def` statement, or by using :term:`generators `. -The :keyword:`async def` type of coroutine was added in Python 3.5, and -is recommended if there is no need to support older Python versions. - -Generator-based coroutines should be decorated with :func:`@asyncio.coroutine -`, although this is not strictly enforced. -The decorator enables compatibility with :keyword:`async def` coroutines, -and also serves as documentation. Generator-based -coroutines use the ``yield from`` syntax introduced in :pep:`380`, -instead of the original ``yield`` syntax. - -The word "coroutine", like the word "generator", is used for two -different (though related) concepts: - -- The function that defines a coroutine - (a function definition using :keyword:`async def` or - decorated with ``@asyncio.coroutine``). If disambiguation is needed - we will call this a *coroutine function* (:func:`iscoroutinefunction` - returns ``True``). - -- The object obtained by calling a coroutine function. This object - represents a computation or an I/O operation (usually a combination) - that will complete eventually. If disambiguation is needed we will - call it a *coroutine object* (:func:`iscoroutine` returns ``True``). - -Things a coroutine can do: - -- ``result = await future`` or ``result = yield from future`` -- - suspends the coroutine until the - future is done, then returns the future's result, or raises an - exception, which will be propagated. (If the future is cancelled, - it will raise a ``CancelledError`` exception.) Note that tasks are - futures, and everything said about futures also applies to tasks. - -- ``result = await coroutine`` or ``result = yield from coroutine`` -- - wait for another coroutine to - produce a result (or raise an exception, which will be propagated). - The ``coroutine`` expression must be a *call* to another coroutine. - -- ``return expression`` -- produce a result to the coroutine that is - waiting for this one using :keyword:`await` or ``yield from``. - -- ``raise exception`` -- raise an exception in the coroutine that is - waiting for this one using :keyword:`await` or ``yield from``. - -Calling a coroutine does not start its code running -- -the coroutine object returned by the call doesn't do anything until you -schedule its execution. There are two basic ways to start it running: -call ``await coroutine`` or ``yield from coroutine`` from another coroutine -(assuming the other coroutine is already running!), or schedule its execution -using the :func:`ensure_future` function or the :meth:`AbstractEventLoop.create_task` -method. +Coroutines declared with async/await syntax is the preferred way of +writing asyncio applications. For example, the following snippet +of code (requires Python 3.7+) prints "hello", waits 1 second, +and then prints "world":: + >>> import asyncio -Coroutines (and tasks) can only run when the event loop is running. + >>> async def main(): + ... print('hello') + ... await asyncio.sleep(1) + ... print('world') -.. decorator:: coroutine + >>> asyncio.run(main()) + hello + world - Decorator to mark generator-based coroutines. This enables - the generator use :keyword:`!yield from` to call :keyword:`async - def` coroutines, and also enables the generator to be called by - :keyword:`async def` coroutines, for instance using an - :keyword:`await` expression. - - There is no need to decorate :keyword:`async def` coroutines themselves. - - If the generator is not yielded from before it is destroyed, an error - message is logged. See :ref:`Detect coroutines never scheduled - `. +Note that simply calling a coroutine will not schedule it to +be executed:: -.. note:: + >>> main() + - In this documentation, some methods are documented as coroutines, - even if they are plain Python functions returning a :class:`Future`. - This is intentional to have a freedom of tweaking the implementation - of these functions in the future. If such a function is needed to be - used in a callback-style code, wrap its result with :func:`ensure_future`. +To actually run a coroutine asyncio provides three main mechanisms: +* The :func:`asyncio.run` function to run the top-level + entry point "main()" function (see the above example.) -.. function:: run(coro, \*, debug=False) +* Awaiting on a coroutine. The following snippet of code will + print "hello" after waiting for 1 second, and then print "world" + after waiting for *another* 2 seconds:: - This function runs the passed coroutine, taking care of - managing the asyncio event loop and finalizing asynchronous - generators. + import asyncio + import time - This function cannot be called when another asyncio event loop is - running in the same thread. + async def say_after(delay, what): + await asyncio.sleep(delay) + print(what) - If debug is True, the event loop will be run in debug mode. + async def main(): + print('started at', time.strftime('%X')) - This function always creates a new event loop and closes it at - the end. It should be used as a main entry point for asyncio - programs, and should ideally only be called once. + await say_after(1, 'hello') + await say_after(2, 'world') - .. versionadded:: 3.7 - **Important:** this has been been added to asyncio in Python 3.7 - on a :term:`provisional basis `. + print('finished at', time.strftime('%X')) + asyncio.run(main()) -.. _asyncio-hello-world-coroutine: + Expected output:: -Example: Hello World coroutine -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + started at 17:13:52 + hello + world + finished at 17:13:55 -Example of coroutine displaying ``"Hello World"``:: +* The :func:`asyncio.create_task` function to run coroutines + concurrently as asyncio :class:`Tasks `. - import asyncio + Let's modify the above example and run two ``say_after`` coroutines + *concurrently*:: + + async def main(): + task1 = asyncio.create_task( + say_after(1, 'hello')) - async def hello_world(): - print("Hello World!") + task2 = asyncio.create_task( + say_after(2, 'world')) - asyncio.run(hello_world()) + print('started at', time.strftime('%X')) -.. seealso:: + # Wait until both tasks are completed (should take + # around 2 seconds.) + await task1 + await task2 - The :ref:`Hello World with call_soon() ` - example uses the :meth:`AbstractEventLoop.call_soon` method to schedule a - callback. + print('finished at', time.strftime('%X')) + Note that expected output now shows that the snippet runs + 1 second faster than before:: -.. _asyncio-date-coroutine: + started at 17:14:32 + hello + world + finished at 17:14:34 -Example: Coroutine displaying the current date -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Example of coroutine displaying the current date every second during 5 seconds -using the :meth:`sleep` function:: +.. _asyncio-awaitables: + +Awaitables +========== + +We say that an object is an **awaitable** object if it can be used +in an :keyword:`await` expression. Many asyncio APIs are designed to +accept awaitables. + +There are three main types of *awaitable* objects: +**coroutines**, **Tasks**, and **Futures**. + + +.. rubric:: Coroutines + +Python coroutines are *awaitables* and therefore can be awaited from +other coroutines:: import asyncio - import datetime - async def display_date(): - loop = asyncio.get_running_loop() - end_time = loop.time() + 5.0 - while True: - print(datetime.datetime.now()) - if (loop.time() + 1.0) >= end_time: - break - await asyncio.sleep(1) + async def nested(): + return 42 - asyncio.run(display_date()) + async def main(): + # Nothing happens if we just call "nested()". + # A coroutine object is created but not awaited, + # so it *won't run at all*. + nested() -.. seealso:: + # Let's do it differently now and await it: + print(await nested()) # will print "42". - The :ref:`display the current date with call_later() - ` example uses a callback with the - :meth:`AbstractEventLoop.call_later` method. + asyncio.run(main()) +.. important:: -Example: Chain coroutines -^^^^^^^^^^^^^^^^^^^^^^^^^ + In this documentation the term "coroutine" can be used for + two closely related concepts: -Example chaining coroutines:: + * a *coroutine function*: an :keyword:`async def` function; - import asyncio + * a *coroutine object*: an object returned by calling a + *coroutine function*. - async def compute(x, y): - print("Compute %s + %s ..." % (x, y)) - await asyncio.sleep(1.0) - return x + y +asyncio also supports legacy :ref:`generator-based +` coroutines. - async def print_sum(x, y): - result = await compute(x, y) - print("%s + %s = %s" % (x, y, result)) - loop = asyncio.get_event_loop() - loop.run_until_complete(print_sum(1, 2)) - loop.close() +.. rubric:: Tasks -``compute()`` is chained to ``print_sum()``: ``print_sum()`` coroutine waits -until ``compute()`` is completed before returning its result. +*Tasks* are used to schedule coroutines *concurrently*. -Sequence diagram of the example: +When a coroutine is wrapped into a *Task* with functions like +:func:`asyncio.create_task` the coroutine is automatically +scheduled to run soon:: -.. image:: tulip_coro.png - :align: center + import asyncio -The "Task" is created by the :meth:`AbstractEventLoop.run_until_complete` method -when it gets a coroutine object instead of a task. + async def nested(): + return 42 -The diagram shows the control flow, it does not describe exactly how things -work internally. For example, the sleep coroutine creates an internal future -which uses :meth:`AbstractEventLoop.call_later` to wake up the task in 1 second. + async def main(): + # Schedule nested() to run soon concurrently + # with "main()". + task = asyncio.create_task(nested()) + # "task" can now be used to cancel "nested()", or + # can simply be awaited to wait until it is complete: + await task -InvalidStateError ------------------ + asyncio.run(main()) -.. exception:: InvalidStateError - The operation is not allowed in this state. +.. rubric:: Futures +A :class:`Future` is a special **low-level** awaitable object that +represents an **eventual result** of an asynchronous operation. -TimeoutError ------------- +When a Future object is *awaited* it means that the coroutine will +wait until the Future is resolved in some other place. -.. exception:: TimeoutError +Future objects in asyncio are needed to allow callback-based code +to be used with async/await. - The operation exceeded the given deadline. +Normally **there is no need** to create Future objects at the +application level code. -.. note:: +Future objects, sometimes exposed by libraries and some asyncio +APIs, can be awaited:: - This exception is different from the builtin :exc:`TimeoutError` exception! + async def main(): + await function_that_returns_a_future_object() + # this is also valid: + await asyncio.gather( + function_that_returns_a_future_object(), + some_python_coroutine() + ) -Future ------- +A good example of a low-level function that returns a Future object +is :meth:`loop.run_in_executor`. -.. class:: Future(\*, loop=None) - This class is *almost* compatible with :class:`concurrent.futures.Future`. +Running an asyncio Program +========================== - Differences: +.. function:: run(coro, \*, debug=False) - - :meth:`result` and :meth:`exception` do not take a timeout argument and - raise an exception when the future isn't done yet. + This function runs the passed coroutine, taking care of + managing the asyncio event loop and *finalizing asynchronous + generators*. - - Callbacks registered with :meth:`add_done_callback` are always called - via the event loop's :meth:`~AbstractEventLoop.call_soon`. + This function cannot be called when another asyncio event loop is + running in the same thread. - - This class is not compatible with the :func:`~concurrent.futures.wait` and - :func:`~concurrent.futures.as_completed` functions in the - :mod:`concurrent.futures` package. + If *debug* is ``True``, the event loop will be run in debug mode. - This class is :ref:`not thread safe `. + This function always creates a new event loop and closes it at + the end. It should be used as a main entry point for asyncio + programs, and should ideally only be called once. - .. method:: cancel() + .. versionadded:: 3.7 + **Important:** this function has been added to asyncio in + Python 3.7 on a :term:`provisional basis `. - Cancel the future and schedule callbacks. - If the future is already done or cancelled, return ``False``. Otherwise, - change the future's state to cancelled, schedule the callbacks and return - ``True``. +Creating Tasks +============== - .. method:: cancelled() +.. function:: create_task(coro) - Return ``True`` if the future was cancelled. + Wrap the *coro* :ref:`coroutine ` into a :class:`Task` + and schedule its execution. Return the Task object. - .. method:: done() + The task is executed in the loop returned by :func:`get_running_loop`, + :exc:`RuntimeError` is raised if there is no running loop in + current thread. - Return ``True`` if the future is done. + This function has been **added in Python 3.7**. Prior to + Python 3.7, the low-level :func:`asyncio.ensure_future` function + can be used instead:: + + async def coro(): + ... + + # In Python 3.7+ + task = asyncio.create_task(coro()) + ... + + # This works in all Python versions but is less readable + task = asyncio.ensure_future(coro()) + ... - Done means either that a result / exception are available, or that the - future was cancelled. + .. versionadded:: 3.7 - .. method:: result() - Return the result this future represents. +Sleeping +======== - If the future has been cancelled, raises :exc:`CancelledError`. If the - future's result isn't yet available, raises :exc:`InvalidStateError`. If - the future is done and has an exception set, this exception is raised. +.. coroutinefunction:: sleep(delay, result=None, \*, loop=None) - .. method:: exception() + Block for *delay* seconds. - Return the exception that was set on this future. + If *result* is provided, it is returned to the caller + when the coroutine completes. - The exception (or ``None`` if no exception was set) is returned only if - the future is done. If the future has been cancelled, raises - :exc:`CancelledError`. If the future isn't done yet, raises - :exc:`InvalidStateError`. + ``sleep()`` always suspends the current task, allowing other tasks + to run. - .. method:: add_done_callback(callback, *, context=None) + The *loop* argument is deprecated and scheduled for removal + in Python 3.10. + + .. _asyncio_example_sleep: + + Example of coroutine displaying the current date every second + for 5 seconds:: + + import asyncio + import datetime - Add a callback to be run when the future becomes done. + async def display_date(): + loop = asyncio.get_running_loop() + end_time = loop.time() + 5.0 + while True: + print(datetime.datetime.now()) + if (loop.time() + 1.0) >= end_time: + break + await asyncio.sleep(1) - The *callback* is called with a single argument - the future object. If the - future is already done when this is called, the callback is scheduled - with :meth:`~AbstractEventLoop.call_soon`. + asyncio.run(display_date()) - An optional keyword-only *context* argument allows specifying a custom - :class:`contextvars.Context` for the *callback* to run in. The current - context is used when no *context* is provided. - :ref:`Use functools.partial to pass parameters to the callback - `. For example, - ``fut.add_done_callback(functools.partial(print, "Future:", - flush=True))`` will call ``print("Future:", fut, flush=True)``. +Running Tasks Concurrently +========================== - .. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. +.. awaitablefunction:: gather(\*aws, loop=None, return_exceptions=False) - .. method:: remove_done_callback(fn) + Run :ref:`awaitable objects ` in the *aws* + sequence *concurrently*. - Remove all instances of a callback from the "call when done" list. + If any awaitable in *aws* is a coroutine, it is automatically + scheduled as a Task. - Returns the number of callbacks removed. + If all awaitables are completed successfully, the result is an + aggregate list of returned values. The order of result values + corresponds to the order of awaitables in *aws*. - .. method:: set_result(result) + If *return_exceptions* is ``False`` (default), the first + raised exception is immediately propagated to the task that + awaits on ``gather()``. Other awaitables in the *aws* sequence + **won't be cancelled** and will continue to run. - Mark the future done and set its result. + If *return_exceptions* is ``True``, exceptions are treated the + same as successful results, and aggregated in the result list. - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. + If ``gather()`` is *cancelled*, all submitted awaitables + (that have not completed yet) are also *cancelled*. - .. method:: set_exception(exception) + If any Task or Future from the *aws* sequence is *cancelled*, it is + treated as if it raised :exc:`CancelledError` -- the ``gather()`` + call is **not** cancelled in this case. This is to prevent the + cancellation of one submitted Task/Future to cause other + Tasks/Futures to be cancelled. - Mark the future done and set an exception. + .. _asyncio_example_gather: - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. + Example:: - .. method:: get_loop() + import asyncio - Return the event loop the future object is bound to. + async def factorial(name, number): + f = 1 + for i in range(2, number + 1): + print(f"Task {name}: Compute factorial({i})...") + await asyncio.sleep(1) + f *= i + print(f"Task {name}: factorial({number}) = {f}") + + async def main(): + # Schedule three calls *concurrently*: + await asyncio.gather( + factorial("A", 2), + factorial("B", 3), + factorial("C", 4), + ) + + asyncio.run(main()) + + # Expected output: + # + # Task A: Compute factorial(2)... + # Task B: Compute factorial(2)... + # Task C: Compute factorial(2)... + # Task A: factorial(2) = 2 + # Task B: Compute factorial(3)... + # Task C: Compute factorial(3)... + # Task B: factorial(3) = 6 + # Task C: Compute factorial(4)... + # Task C: factorial(4) = 24 - .. versionadded:: 3.7 + .. versionchanged:: 3.7 + If the *gather* itself is cancelled, the cancellation is + propagated regardless of *return_exceptions*. -Example: Future with run_until_complete() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Shielding From Cancellation +=========================== -Example combining a :class:`Future` and a :ref:`coroutine function -`:: +.. awaitablefunction:: shield(aw, \*, loop=None) - import asyncio + Protect an :ref:`awaitable object ` + from being :meth:`cancelled `. - async def slow_operation(future): - await asyncio.sleep(1) - future.set_result('Future is done!') - - loop = asyncio.get_event_loop() - future = asyncio.Future() - asyncio.ensure_future(slow_operation(future)) - loop.run_until_complete(future) - print(future.result()) - loop.close() - -The coroutine function is responsible for the computation (which takes 1 second) -and it stores the result into the future. The -:meth:`~AbstractEventLoop.run_until_complete` method waits for the completion of -the future. + If *aw* is a coroutine it is automatically scheduled as a Task. -.. note:: - The :meth:`~AbstractEventLoop.run_until_complete` method uses internally the - :meth:`~Future.add_done_callback` method to be notified when the future is - done. + The statement:: + + res = await shield(something()) + is equivalent to:: -Example: Future with run_forever() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + res = await something() -The previous example can be written differently using the -:meth:`Future.add_done_callback` method to describe explicitly the control -flow:: + *except* that if the coroutine containing it is cancelled, the + Task running in ``something()`` is not cancelled. From the point + of view of ``something()``, the cancellation did not happen. + Although its caller is still cancelled, so the "await" expression + still raises a :exc:`CancelledError`. + + If ``something()`` is cancelled by other means (i.e. from within + itself) that would also cancel ``shield()``. + + If it is desired to completely ignore cancellation (not recommended) + the ``shield()`` function should be combined with a try/except + clause, as follows:: - import asyncio + try: + res = await shield(something()) + except CancelledError: + res = None - async def slow_operation(future): - await asyncio.sleep(1) - future.set_result('Future is done!') - - def got_result(future): - print(future.result()) - loop.stop() - - loop = asyncio.get_event_loop() - future = asyncio.Future() - asyncio.ensure_future(slow_operation(future)) - future.add_done_callback(got_result) - try: - loop.run_forever() - finally: - loop.close() - -In this example, the future is used to link ``slow_operation()`` to -``got_result()``: when ``slow_operation()`` is done, ``got_result()`` is called -with the result. +Timeouts +======== -Task ----- +.. coroutinefunction:: wait_for(aw, timeout, \*, loop=None) -.. function:: create_task(coro) + Wait for the *aw* :ref:`awaitable ` + to complete with a timeout. - Wrap a :ref:`coroutine ` *coro* into a task and schedule - its execution. Return the task object. + If *aw* is a coroutine it is automatically scheduled as a Task. - The task is executed in :func:`get_running_loop` context, - :exc:`RuntimeError` is raised if there is no running loop in - current thread. + *timeout* can either be ``None`` or a float or int number of seconds + to wait for. If *timeout* is ``None``, block until the future + completes. - .. versionadded:: 3.7 + If a timeout occurs, it cancels the task and raises + :exc:`asyncio.TimeoutError`. -.. class:: Task(coro, \*, loop=None) + To avoid the task :meth:`cancellation `, + wrap it in :func:`shield`. - A unit for concurrent running of :ref:`coroutines `, - subclass of :class:`Future`. + The function will wait until the future is actually cancelled, + so the total wait time may exceed the *timeout*. - A task is responsible for executing a coroutine object in an event loop. If - the wrapped coroutine yields from a future, the task suspends the execution - of the wrapped coroutine and waits for the completion of the future. When - the future is done, the execution of the wrapped coroutine restarts with the - result or the exception of the future. - - Event loops use cooperative scheduling: an event loop only runs one task at - a time. Other tasks may run in parallel if other event loops are - running in different threads. While a task waits for the completion of a - future, the event loop executes a new task. - - The cancellation of a task is different from the cancellation of a - future. Calling :meth:`cancel` will throw a - :exc:`~concurrent.futures.CancelledError` to the wrapped - coroutine. :meth:`~Future.cancelled` only returns ``True`` if the - wrapped coroutine did not catch the - :exc:`~concurrent.futures.CancelledError` exception, or raised a - :exc:`~concurrent.futures.CancelledError` exception. - - If a pending task is destroyed, the execution of its wrapped :ref:`coroutine - ` did not complete. It is probably a bug and a warning is - logged: see :ref:`Pending task destroyed `. + If the wait is cancelled, the future *aw* is also cancelled. - Don't directly create :class:`Task` instances: use the :func:`create_task` - function or the :meth:`AbstractEventLoop.create_task` method. + The *loop* argument is deprecated and scheduled for removal + in Python 3.10. - Tasks support the :mod:`contextvars` module. When a Task - is created it copies the current context and later runs its coroutine - in the copied context. See :pep:`567` for more details. + .. _asyncio_example_waitfor: - This class is :ref:`not thread safe `. + Example:: + + async def eternity(): + # Sleep for one hour + await asyncio.sleep(3600) + print('yay!') + + async def main(): + # Wait for at most 1 second + try: + await asyncio.wait_for(eternity(), timeout=1.0) + except asyncio.TimeoutError: + print('timeout!') + + asyncio.run(main()) + + # Expected output: + # + # timeout! .. versionchanged:: 3.7 - Added support for the :mod:`contextvars` module. + When *aw* is cancelled due to a timeout, ``wait_for`` waits + for *aw* to be cancelled. Previously, it raised + :exc:`asyncio.TimeoutError` immediately. - .. classmethod:: all_tasks(loop=None) - Return a set of all tasks for an event loop. +Waiting Primitives +================== - By default all tasks for the current event loop are returned. - If *loop* is ``None``, :func:`get_event_loop` function - is used to get the current loop. +.. coroutinefunction:: wait(aws, \*, loop=None, timeout=None,\ + return_when=ALL_COMPLETED) - .. classmethod:: current_task(loop=None) + Run :ref:`awaitable objects ` in the *aws* + set concurrently and block until the condition specified + by *return_when*. + + If any awaitable in *aws* is a coroutine, it is automatically + scheduled as a Task. Passing coroutines objects to + ``wait()`` directly is deprecated as it leads to + :ref:`confusing behavior `. - Return the currently running task in an event loop or ``None``. + Returns two sets of Tasks/Futures: ``(done, pending)``. - By default the current task for the current event loop is returned. + Usage:: - ``None`` is returned when called not in the context of a :class:`Task`. + done, pending = await asyncio.wait(aws) - .. method:: cancel() + The *loop* argument is deprecated and scheduled for removal + in Python 3.10. - Request that this task cancel itself. + *timeout* (a float or int), if specified, can be used to control + the maximum number of seconds to wait before returning. - This arranges for a :exc:`~concurrent.futures.CancelledError` to be - thrown into the wrapped coroutine on the next cycle through the event - loop. The coroutine then has a chance to clean up or even deny the - request using try/except/finally. - - Unlike :meth:`Future.cancel`, this does not guarantee that the task - will be cancelled: the exception might be caught and acted upon, delaying - cancellation of the task or preventing cancellation completely. The task - may also return a value or raise a different exception. - - Immediately after this method is called, :meth:`~Future.cancelled` will - not return ``True`` (unless the task was already cancelled). A task will - be marked as cancelled when the wrapped coroutine terminates with a - :exc:`~concurrent.futures.CancelledError` exception (even if - :meth:`cancel` was not called). + Note that this function does not raise :exc:`asyncio.TimeoutError`. + Futures or Tasks that aren't done when the timeout occurs are simply + returned in the second set. - .. method:: get_stack(\*, limit=None) + *return_when* indicates when this function should return. It must + be one of the following constants: - Return the list of stack frames for this task's coroutine. + .. tabularcolumns:: |l|L| - If the coroutine is not done, this returns the stack where it is - suspended. If the coroutine has completed successfully or was - cancelled, this returns an empty list. If the coroutine was - terminated by an exception, this returns the list of traceback - frames. + +-----------------------------+----------------------------------------+ + | Constant | Description | + +=============================+========================================+ + | :const:`FIRST_COMPLETED` | The function will return when any | + | | future finishes or is cancelled. | + +-----------------------------+----------------------------------------+ + | :const:`FIRST_EXCEPTION` | The function will return when any | + | | future finishes by raising an | + | | exception. If no future raises an | + | | exception then it is equivalent to | + | | :const:`ALL_COMPLETED`. | + +-----------------------------+----------------------------------------+ + | :const:`ALL_COMPLETED` | The function will return when all | + | | futures finish or are cancelled. | + +-----------------------------+----------------------------------------+ - The frames are always ordered from oldest to newest. + Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the + futures when a timeout occurs. + + .. _asyncio_example_wait_coroutine: + .. note:: - The optional limit gives the maximum number of frames to return; by - default all available frames are returned. Its meaning differs depending - on whether a stack or a traceback is returned: the newest frames of a - stack are returned, but the oldest frames of a traceback are returned. - (This matches the behavior of the traceback module.) + ``wait()`` schedules coroutines as Tasks automatically and later + returns those implicitly created Task objects in ``(done, pending)`` + sets. Therefore the following code won't work as expected:: - For reasons beyond our control, only one stack frame is returned for a - suspended coroutine. + async def foo(): + return 42 - .. method:: print_stack(\*, limit=None, file=None) + coro = foo() + done, pending = await asyncio.wait({coro}) - Print the stack or traceback for this task's coroutine. + if coro in done: + # This branch will never be run! - This produces output similar to that of the traceback module, for the - frames retrieved by get_stack(). The limit argument is passed to - get_stack(). The file argument is an I/O stream to which the output - is written; by default output is written to sys.stderr. + Here is how the above snippet can be fixed:: + async def foo(): + return 42 -Example: Parallel execution of tasks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + task = asyncio.create_task(foo()) + done, pending = await asyncio.wait({task}) -Example executing 3 tasks (A, B, C) in parallel:: + if task in done: + # Everything will work as expected now. - import asyncio + Passing coroutine objects to ``wait()`` directly is + deprecated. - async def factorial(name, number): - f = 1 - for i in range(2, number+1): - print("Task %s: Compute factorial(%s)..." % (name, i)) - await asyncio.sleep(1) - f *= i - print("Task %s: factorial(%s) = %s" % (name, number, f)) - loop = asyncio.get_event_loop() - loop.run_until_complete(asyncio.gather( - factorial("A", 2), - factorial("B", 3), - factorial("C", 4), - )) - loop.close() - -Output:: - - Task A: Compute factorial(2)... - Task B: Compute factorial(2)... - Task C: Compute factorial(2)... - Task A: factorial(2) = 2 - Task B: Compute factorial(3)... - Task C: Compute factorial(3)... - Task B: factorial(3) = 6 - Task C: Compute factorial(4)... - Task C: factorial(4) = 24 +.. function:: as_completed(aws, \*, loop=None, timeout=None) -A task is automatically scheduled for execution when it is created. The event -loop stops when all tasks are done. + Run :ref:`awaitable objects ` in the *aws* + set concurrently. Return an iterator of :class:`Future` objects. + Each Future object returned represents the earliest result + from the set of the remaining awaitables. + Raises :exc:`asyncio.TimeoutError` if the timeout occurs before + all Futures are done. -Task functions --------------- + Example:: -.. note:: + for f in as_completed(aws): + earliest_result = await f + # ... + + +Scheduling From Other Threads +============================= + +.. function:: run_coroutine_threadsafe(coro, loop) + + Submit a coroutine to the given event loop. Thread-safe. + + Return a :class:`concurrent.futures.Future` to wait for the result + from another OS thread. + + This function is meant to be called from a different OS thread + than the one where the event loop is running. Example:: + + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) - In the functions below, the optional *loop* argument allows explicitly setting - the event loop object used by the underlying task or coroutine. If it's - not provided, the default event loop is used. + # Wait for the result with an optional timeout argument + assert future.result(timeout) == 3 + + If an exception is raised in the coroutine, the returned Future + will be notified. It can also be used to cancel the task in + the event loop:: + + try: + result = future.result(timeout) + except asyncio.TimeoutError: + print('The coroutine took too long, cancelling the task...') + future.cancel() + except Exception as exc: + print('The coroutine raised an exception: {!r}'.format(exc)) + else: + print('The coroutine returned: {!r}'.format(result)) + + See the :ref:`concurrency and multithreading ` + section of the documentation. + + Unlike other asyncio functions this functions requires the *loop* + argument to be passed explicitly. + + .. versionadded:: 3.5.1 + + +Introspection +============= .. function:: current_task(loop=None) - Return the current running :class:`Task` instance or ``None``, if + Return the currently running :class:`Task` instance, or ``None`` if no task is running. If *loop* is ``None`` :func:`get_running_loop` is used to get @@ -567,246 +627,282 @@ .. function:: all_tasks(loop=None) - Return a set of :class:`Task` objects created for the loop. + Return a set of not yet finished :class:`Task` objects run by + the loop. If *loop* is ``None``, :func:`get_running_loop` is used for getting - current loop (contrary to the deprecated :meth:`Task.all_tasks` method - that uses :func:`get_event_loop`.) + current loop. .. versionadded:: 3.7 -.. function:: as_completed(fs, \*, loop=None, timeout=None) +Task Object +=========== - Return an iterator whose values, when waited for, are :class:`Future` - instances. +.. class:: Task(coro, \*, loop=None) - Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures - are done. + A :class:`Future-like ` object that runs a Python + :ref:`coroutine `. Not thread-safe. - Example:: + Tasks are used to run coroutines in event loops. + If a coroutine awaits on a Future, the Task suspends + the execution of the coroutine and waits for the completion + of the Future. When the Future is *done*, the execution of + the wrapped coroutine resumes. + + Event loops use cooperative scheduling: an event loop runs + one Task at a time. While a Task awaits for the completion of a + Future, the event loop runs other Tasks, callbacks, or performs + IO operations. + + Use the high-level :func:`asyncio.create_task` function to create + Tasks, or the low-level :meth:`loop.create_task` or + :func:`ensure_future` functions. Manual instantiation of Tasks + is discouraged. + + To cancel a running Task use the :meth:`cancel` method. Calling it + will cause the Task to throw a :exc:`CancelledError` exception into + the wrapped coroutine. If a coroutine is awaiting on a Future + object during cancellation, the Future object will be cancelled. + + :meth:`cancelled` can be used to check if the Task was cancelled. + The method returns ``True`` if the wrapped coroutine did not + suppress the :exc:`CancelledError` exception and was actually + cancelled. + + :class:`asyncio.Task` inherits from :class:`Future` all of its + APIs except :meth:`Future.set_result` and + :meth:`Future.set_exception`. - for f in as_completed(fs): - result = await f # The 'await' may raise - # Use result + Tasks support the :mod:`contextvars` module. When a Task + is created it copies the current context and later runs its + coroutine in the copied context. - .. note:: + .. versionchanged:: 3.7 + Added support for the :mod:`contextvars` module. - The futures ``f`` are not necessarily members of fs. + .. method:: cancel() -.. function:: ensure_future(coro_or_future, \*, loop=None) + Request the Task to be cancelled. - Schedule the execution of a :ref:`coroutine object `: wrap it in - a future. Return a :class:`Task` object. + This arranges for a :exc:`CancelledError` exception to be thrown + into the wrapped coroutine on the next cycle of the event loop. - If the argument is a :class:`Future`, it is returned directly. + The coroutine then has a chance to clean up or even deny the + request by suppressing the exception with a :keyword:`try` ... + ... ``except CancelledError`` ... :keyword:`finally` block. + Therefore, unlike :meth:`Future.cancel`, :meth:`Task.cancel` does + not guarantee that the Task will be cancelled, although + suppressing cancellation completely is not common and is actively + discouraged. + + .. _asyncio_example_task_cancel: + + The following example illustrates how coroutines can intercept + the cancellation request:: + + async def cancel_me(): + print('cancel_me(): before sleep') + + try: + # Wait for 1 hour + await asyncio.sleep(3600) + except asyncio.CancelledError: + print('cancel_me(): cancel sleep') + raise + finally: + print('cancel_me(): after sleep') + + async def main(): + # Create a "cancel_me" Task + task = asyncio.create_task(cancel_me()) + + # Wait for 1 second + await asyncio.sleep(1) + + task.cancel() + try: + await task + except asyncio.CancelledError: + print("main(): cancel_me is cancelled now") + + asyncio.run(main()) + + # Expected output: + # + # cancel_me(): before sleep + # cancel_me(): cancel sleep + # cancel_me(): after sleep + # main(): cancel_me is cancelled now - .. versionadded:: 3.4.4 + .. method:: cancelled() - .. versionchanged:: 3.5.1 - The function accepts any :term:`awaitable` object. + Return ``True`` if the Task is *cancelled*. - .. note:: + The Task is *cancelled* when the cancellation was requested with + :meth:`cancel` and the wrapped coroutine propagated the + :exc:`CancelledError` exception thrown into it. - :func:`create_task` (added in Python 3.7) is the preferable way - for spawning new tasks. + .. method:: done() - .. seealso:: + Return ``True`` if the Task is *done*. - The :func:`create_task` function and - :meth:`AbstractEventLoop.create_task` method. + A Task is *done* when the wrapped coroutine either returned + a value, raised an exception, or the Task was cancelled. -.. function:: wrap_future(future, \*, loop=None) + .. method:: result() - Wrap a :class:`concurrent.futures.Future` object in a :class:`Future` - object. + Return the result of the Task. -.. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) + If the Task is *done*, the result of the wrapped coroutine + is returned (or if the coroutine raised an exception, that + exception is re-raised.) - Return a future aggregating results from the given coroutine objects or - futures. + If the Task has been *cancelled*, this method raises + a :exc:`CancelledError` exception. - All futures must share the same event loop. If all the tasks are done - successfully, the returned future's result is the list of results (in the - order of the original sequence, not necessarily the order of results - arrival). If *return_exceptions* is true, exceptions in the tasks are - treated the same as successful results, and gathered in the result list; - otherwise, the first raised exception will be immediately propagated to the - returned future. + If the Task's result isn't yet available, this method raises + a :exc:`InvalidStateError` exception. - Cancellation: if the outer Future is cancelled, all children (that have not - completed yet) are also cancelled. If any child is cancelled, this is - treated as if it raised :exc:`~concurrent.futures.CancelledError` -- the - outer Future is *not* cancelled in this case. (This is to prevent the - cancellation of one child to cause other children to be cancelled.) + .. method:: exception() - .. versionchanged:: 3.7.0 - If the *gather* itself is cancelled, the cancellation is propagated - regardless of *return_exceptions*. + Return the exception of the Task. -.. function:: iscoroutine(obj) + If the wrapped coroutine raised an exception that exception + is returned. If the wrapped coroutine returned normally + this method returns ``None``. - Return ``True`` if *obj* is a :ref:`coroutine object `, - which may be based on a generator or an :keyword:`async def` coroutine. + If the Task has been *cancelled*, this method raises a + :exc:`CancelledError` exception. -.. function:: iscoroutinefunction(func) + If the Task isn't *done* yet, this method raises an + :exc:`InvalidStateError` exception. - Return ``True`` if *func* is determined to be a :ref:`coroutine function - `, which may be a decorated generator function or an - :keyword:`async def` function. + .. method:: add_done_callback(callback, *, context=None) -.. function:: run_coroutine_threadsafe(coro, loop) + Add a callback to be run when the Task is *done*. - Submit a :ref:`coroutine object ` to a given event loop. + This method should only be used in low-level callback-based code. - Return a :class:`concurrent.futures.Future` to access the result. + See the documentation of :meth:`Future.add_done_callback` + for more details. - This function is meant to be called from a different thread than the one - where the event loop is running. Usage:: + .. method:: remove_done_callback(callback) - # Create a coroutine - coro = asyncio.sleep(1, result=3) - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 + Remove *callback* from the callbacks list. - If an exception is raised in the coroutine, the returned future will be - notified. It can also be used to cancel the task in the event loop:: + This method should only be used in low-level callback-based code. - try: - result = future.result(timeout) - except asyncio.TimeoutError: - print('The coroutine took too long, cancelling the task...') - future.cancel() - except Exception as exc: - print('The coroutine raised an exception: {!r}'.format(exc)) - else: - print('The coroutine returned: {!r}'.format(result)) + See the documentation of :meth:`Future.remove_done_callback` + for more details. - See the :ref:`concurrency and multithreading ` - section of the documentation. + .. method:: get_stack(\*, limit=None) - .. note:: + Return the list of stack frames for this Task. - Unlike other functions from the module, - :func:`run_coroutine_threadsafe` requires the *loop* argument to - be passed explicitly. + If the wrapped coroutine is not done, this returns the stack + where it is suspended. If the coroutine has completed + successfully or was cancelled, this returns an empty list. + If the coroutine was terminated by an exception, this returns + the list of traceback frames. - .. versionadded:: 3.5.1 + The frames are always ordered from oldest to newest. -.. coroutinefunction:: sleep(delay, result=None, \*, loop=None) + Only one stack frame is returned for a suspended coroutine. - Create a :ref:`coroutine ` that completes after a given - time (in seconds). If *result* is provided, it is produced to the caller - when the coroutine completes. + The optional *limit* argument sets the maximum number of frames + to return; by default all available frames are returned. + The ordering of the returned list differs depending on whether + a stack or a traceback is returned: the newest frames of a + stack are returned, but the oldest frames of a traceback are + returned. (This matches the behavior of the traceback module.) - The resolution of the sleep depends on the :ref:`granularity of the event - loop `. + .. method:: print_stack(\*, limit=None, file=None) - This function is a :ref:`coroutine `. + Print the stack or traceback for this Task. -.. coroutinefunction:: shield(arg, \*, loop=None) + This produces output similar to that of the traceback module + for the frames retrieved by :meth:`get_stack`. - Wait for a future, shielding it from cancellation. + The *limit* argument is passed to :meth:`get_stack` directly. - The statement:: + The *file* argument is an I/O stream to which the output + is written; by default output is written to :data:`sys.stderr`. - res = await shield(something()) + .. classmethod:: all_tasks(loop=None) - is exactly equivalent to the statement:: + Return a set of all tasks for an event loop. - res = await something() + By default all tasks for the current event loop are returned. + If *loop* is ``None``, the :func:`get_event_loop` function + is used to get the current loop. - *except* that if the coroutine containing it is cancelled, the task running - in ``something()`` is not cancelled. From the point of view of - ``something()``, the cancellation did not happen. But its caller is still - cancelled, so the yield-from expression still raises - :exc:`~concurrent.futures.CancelledError`. Note: If ``something()`` is - cancelled by other means this will still cancel ``shield()``. + This method is **deprecated** and will be removed in + Python 3.9. Use the :func:`asyncio.all_tasks` function instead. - If you want to completely ignore cancellation (not recommended) you can - combine ``shield()`` with a try/except clause, as follows:: + .. classmethod:: current_task(loop=None) - try: - res = await shield(something()) - except CancelledError: - res = None + Return the currently running task or ``None``. + If *loop* is ``None``, the :func:`get_event_loop` function + is used to get the current loop. -.. coroutinefunction:: wait(futures, \*, loop=None, timeout=None,\ - return_when=ALL_COMPLETED) + This method is **deprecated** and will be removed in + Python 3.9. Use the :func:`asyncio.current_task` function + instead. - Wait for the Futures and coroutine objects given by the sequence *futures* - to complete. Coroutines will be wrapped in Tasks. Returns two sets of - :class:`Future`: (done, pending). - - The sequence *futures* must not be empty. - - *timeout* can be used to control the maximum number of seconds to wait before - returning. *timeout* can be an int or float. If *timeout* is not specified - or ``None``, there is no limit to the wait time. - *return_when* indicates when this function should return. It must be one of - the following constants of the :mod:`concurrent.futures` module: +.. _asyncio_generator_based_coro: - .. tabularcolumns:: |l|L| +Generator-based Coroutines +========================== - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ +.. note:: - Unlike :func:`~asyncio.wait_for`, ``wait()`` will not cancel the futures - when a timeout occurs. + Support for generator-based coroutines is **deprecated** and + is scheduled for removal in Python 3.10. - This function is a :ref:`coroutine `. +Generator-based coroutines predate async/await syntax. They are +Python generators that use ``yield from`` expressions to await +on Futures and other coroutines. + +Generator-based coroutines should be decorated with +:func:`@asyncio.coroutine `, although this is not +enforced. - Usage:: - done, pending = await asyncio.wait(fs) +.. decorator:: coroutine - .. note:: + Decorator to mark generator-based coroutines. - This does not raise :exc:`asyncio.TimeoutError`! Futures that aren't done - when the timeout occurs are returned in the second set. + This decorator enables legacy generator-based coroutines to be + compatible with async/await code:: + @asyncio.coroutine + def old_style_coroutine(): + yield from asyncio.sleep(1) -.. coroutinefunction:: wait_for(fut, timeout, \*, loop=None) + async def main(): + await old_style_coroutine() - Wait for the single :class:`Future` or :ref:`coroutine object ` - to complete with timeout. If *timeout* is ``None``, block until the future - completes. + This decorator is **deprecated** and is scheduled for removal in + Python 3.10. - Coroutine will be wrapped in :class:`Task`. + This decorator should not be used for :keyword:`async def` + coroutines. - Returns result of the Future or coroutine. When a timeout occurs, it - cancels the task and raises :exc:`asyncio.TimeoutError`. To avoid the task - cancellation, wrap it in :func:`shield`. The function will wait until - the future is actually cancelled, so the total wait time may exceed - the *timeout*. +.. function:: iscoroutine(obj) - If the wait is cancelled, the future *fut* is also cancelled. + Return ``True`` if *obj* is a :ref:`coroutine object `. - This function is a :ref:`coroutine `, usage:: + This method is different from :func:`inspect.iscoroutine` because + it returns ``True`` for generator-based coroutines. - result = await asyncio.wait_for(fut, 60.0) +.. function:: iscoroutinefunction(func) - .. versionchanged:: 3.4.3 - If the wait is cancelled, the future *fut* is now also cancelled. + Return ``True`` if *func* is a :ref:`coroutine function + `. - .. versionchanged:: 3.7 - When *fut* is cancelled due to a timeout, ``wait_for`` now waits - for *fut* to be cancelled. Previously, - it raised :exc:`~asyncio.TimeoutError` immediately. + This method is different from :func:`inspect.iscoroutinefunction` + because it returns ``True`` for generator-based coroutine functions + decorated with :func:`@coroutine `. diff -Nru python3.7-3.7.0/Doc/library/collections.rst python3.7-3.7.1/Doc/library/collections.rst --- python3.7-3.7.0/Doc/library/collections.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/collections.rst 2018-10-20 06:04:19.000000000 +0000 @@ -198,6 +198,7 @@ >>> d['lion'] = 'orange' # update an existing key two levels down >>> d['snake'] = 'red' # new keys get added to the topmost dict >>> del d['elephant'] # remove an existing key one level down + >>> d # display result DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) diff -Nru python3.7-3.7.0/Doc/library/concurrent.futures.rst python3.7-3.7.1/Doc/library/concurrent.futures.rst --- python3.7-3.7.0/Doc/library/concurrent.futures.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/concurrent.futures.rst 2018-10-20 06:04:19.000000000 +0000 @@ -223,7 +223,7 @@ *initializer* is an optional callable that is called at the start of each worker process; *initargs* is a tuple of arguments passed to the initializer. Should *initializer* raise an exception, all currently - pending jobs will raise a :exc:`~concurrent.futures.thread.BrokenThreadPool`, + pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`, as well any attempt to submit more jobs to the pool. .. versionchanged:: 3.3 diff -Nru python3.7-3.7.0/Doc/library/configparser.rst python3.7-3.7.1/Doc/library/configparser.rst --- python3.7-3.7.0/Doc/library/configparser.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/configparser.rst 2018-10-20 06:04:19.000000000 +0000 @@ -995,16 +995,17 @@ .. method:: read(filenames, encoding=None) - Attempt to read and parse a list of filenames, returning a list of + Attempt to read and parse an iterable of filenames, returning a list of filenames which were successfully parsed. If *filenames* is a string, a :class:`bytes` object or a :term:`path-like object`, it is treated as a single filename. If a file named in *filenames* cannot be opened, that - file will be ignored. This is designed so that you can specify a list of - potential configuration file locations (for example, the current - directory, the user's home directory, and some system-wide directory), - and all existing configuration files in the list will be read. + file will be ignored. This is designed so that you can specify an + iterable of potential configuration file locations (for example, the + current directory, the user's home directory, and some system-wide + directory), and all existing configuration files in the iterable will be + read. If none of the named files exist, the :class:`ConfigParser` instance will contain an empty dataset. An application which requires diff -Nru python3.7-3.7.0/Doc/library/contextlib.rst python3.7-3.7.1/Doc/library/contextlib.rst --- python3.7-3.7.0/Doc/library/contextlib.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/contextlib.rst 2018-10-20 06:04:19.000000000 +0000 @@ -47,22 +47,28 @@ function for :keyword:`with` statement context managers, without needing to create a class or separate :meth:`__enter__` and :meth:`__exit__` methods. - A simple example (this is not recommended as a real way of generating HTML!):: + While many objects natively support use in with statements, sometimes a + resource needs to be managed that isn't a context manager in its own right, + and doesn't implement a ``close()`` method for use with ``contextlib.closing`` + + An abstract example would be the following to ensure correct resource + management:: from contextlib import contextmanager @contextmanager - def tag(name): - print("<%s>" % name) - yield - print("" % name) + def managed_resource(*args, **kwds): + # Code to acquire resource, e.g.: + resource = acquire_resource(*args, **kwds) + try: + yield resource + finally: + # Code to release resource, e.g.: + release_resource(resource) - >>> with tag("h1"): - ... print("foo") - ... -

- foo -

+ >>> with managed_resource(timeout=3600) as resource: + ... # Resource is released at the end of this block, + ... # even if code in the block raises an exception The function being decorated must return a :term:`generator`-iterator when called. This iterator must yield exactly one value, which will be bound to @@ -110,7 +116,7 @@ async def get_connection(): conn = await acquire_db_connection() try: - yield + yield conn finally: await release_db_connection(conn) @@ -152,10 +158,22 @@ .. function:: nullcontext(enter_result=None) - Return a context manager that returns enter_result from ``__enter__``, but + Return a context manager that returns *enter_result* from ``__enter__``, but otherwise does nothing. It is intended to be used as a stand-in for an optional context manager, for example:: + def myfunction(arg, ignore_exceptions=False): + if ignore_exceptions: + # Use suppress to ignore all exceptions. + cm = contextlib.suppress(Exception) + else: + # Do not ignore any exceptions, cm has no effect. + cm = contextlib.nullcontext() + with cm: + # Do something + + An example using *enter_result*:: + def process_file(file_or_path): if isinstance(file_or_path, str): # If string, open file @@ -453,11 +471,11 @@ .. method:: push_async_exit(exit) Similar to :meth:`push` but expects either an asynchronous context manager - or a coroutine. + or a coroutine function. .. method:: push_async_callback(callback, *args, **kwds) - Similar to :meth:`callback` but expects a coroutine. + Similar to :meth:`callback` but expects a coroutine function. .. method:: aclose() diff -Nru python3.7-3.7.0/Doc/library/contextvars.rst python3.7-3.7.1/Doc/library/contextvars.rst --- python3.7-3.7.0/Doc/library/contextvars.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/contextvars.rst 2018-10-20 06:04:19.000000000 +0000 @@ -48,6 +48,8 @@ The name of the variable. This is a read-only property. + .. versionadded:: 3.7.1 + .. method:: get([default]) Return a value for the context variable for the current context. diff -Nru python3.7-3.7.0/Doc/library/ctypes.rst python3.7-3.7.1/Doc/library/ctypes.rst --- python3.7-3.7.0/Doc/library/ctypes.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/ctypes.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1025,6 +1025,22 @@ 1 5 7 33 99 >>> +The function factories can be used as decorator factories, so we may as well +write:: + + >>> @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int)) + ... def py_cmp_func(a, b): + ... print("py_cmp_func", a[0], b[0]) + ... return a[0] - b[0] + ... + >>> qsort(ia, len(ia), sizeof(c_int), py_cmp_func) + py_cmp_func 5 1 + py_cmp_func 33 99 + py_cmp_func 7 33 + py_cmp_func 1 7 + py_cmp_func 5 7 + >>> + .. note:: Make sure you keep references to :func:`CFUNCTYPE` objects as long as they @@ -1577,7 +1593,9 @@ Function prototypes are similar to function prototypes in C; they describe a function (return type, argument types, calling convention) without defining an implementation. The factory functions must be called with the desired result -type and the argument types of the function. +type and the argument types of the function, and can be used as decorator +factories, and as such, be applied to functions through the ``@wrapper`` syntax. +See :ref:`ctypes-callback-functions` for examples. .. function:: CFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) diff -Nru python3.7-3.7.0/Doc/library/dataclasses.rst python3.7-3.7.1/Doc/library/dataclasses.rst --- python3.7-3.7.0/Doc/library/dataclasses.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/dataclasses.rst 2018-10-20 06:04:19.000000000 +0000 @@ -60,8 +60,9 @@ The :func:`dataclass` decorator will add various "dunder" methods to the class, described below. If any of the added methods already - exist on the class, a :exc:`TypeError` will be raised. The decorator - returns the same class that is called on: no new class is created. + exist on the class, the behavior depends on the parameter, as documented + below. The decorator returns the same class that is called on; no new + class is created. If :func:`dataclass` is used just as a simple decorator with no parameters, it acts as if it has the default values documented in this @@ -115,7 +116,7 @@ If the class already defines any of :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then - :exc:`ValueError` is raised. + :exc:`TypeError` is raised. - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method is generated according to how ``eq`` and ``frozen`` are set. @@ -308,7 +309,7 @@ Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. -.. function:: astuple(*, tuple_factory=tuple) +.. function:: astuple(instance, *, tuple_factory=tuple) Converts the dataclass ``instance`` to a tuple (by using the factory function ``tuple_factory``). Each dataclass is converted @@ -447,7 +448,7 @@ the optional :meth:`__post_init__` method. They are not otherwise used by dataclasses. -For example, suppose a field will be initialzed from a database, if a +For example, suppose a field will be initialized from a database, if a value is not provided when creating the class:: @dataclass @@ -532,7 +533,7 @@ class C: x = [] def add(self, element): - self.x += element + self.x.append(element) o1 = C() o2 = C() diff -Nru python3.7-3.7.0/Doc/library/datetime.rst python3.7-3.7.1/Doc/library/datetime.rst --- python3.7-3.7.0/Doc/library/datetime.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/datetime.rst 2018-10-20 06:04:19.000000000 +0000 @@ -241,7 +241,7 @@ +--------------------------------+-----------------------------------------------+ | ``t1 = t2 - t3`` | Difference of *t2* and *t3*. Afterwards *t1* | | | == *t2* - *t3* and *t2* == *t1* + *t3* are | -| | true. (1) | +| | true. (1)(6) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 * i or t1 = i * t2`` | Delta multiplied by an integer. | | | Afterwards *t1* // i == *t2* is true, | @@ -317,6 +317,11 @@ >>> print(_) -1 day, 19:00:00 +(6) + The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except + when t3 is equal to ``timedelta.max``; in that case the former will produce a result + while the latter will overflow. + In addition to the operations listed above :class:`timedelta` objects support certain additions and subtractions with :class:`date` and :class:`.datetime` objects (see below). @@ -513,8 +518,6 @@ :const:`MINYEAR` or larger than :const:`MAXYEAR`. (2) - This isn't quite equivalent to date1 + (-timedelta), because -timedelta in - isolation can overflow in cases where date1 - timedelta does not. ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. (3) @@ -523,10 +526,9 @@ (4) In other words, ``date1 < date2`` if and only if ``date1.toordinal() < - date2.toordinal()``. In order to stop comparison from falling back to the - default scheme of comparing object addresses, date comparison normally raises - :exc:`TypeError` if the other comparand isn't also a :class:`date` object. - However, ``NotImplemented`` is returned instead if the other comparand has a + date2.toordinal()``. Date comparison raises :exc:`TypeError` if + the other comparand isn't also a :class:`date` object. However, + ``NotImplemented`` is returned instead if the other comparand has a :meth:`timetuple` attribute. This hook gives other kinds of date objects a chance at implementing mixed-type comparison. If not, when a :class:`date` object is compared to an object of a different type, :exc:`TypeError` is raised @@ -839,7 +841,7 @@ Return a :class:`datetime` corresponding to a *date_string* in one of the formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`. Specifically, this function supports strings in the format(s) - ``YYYY-MM-DD[*HH[:MM[:SS[.mmm[mmm]]]][+HH:MM[:SS[.ffffff]]]]``, + ``YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]``, where ``*`` can match any single character. .. caution:: @@ -961,8 +963,6 @@ Computes the datetime2 such that datetime2 + timedelta == datetime1. As for addition, the result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and no time zone adjustments are done even if the input is aware. - This isn't quite equivalent to datetime1 + (-timedelta), because -timedelta - in isolation can overflow in cases where datetime1 - timedelta does not. (3) Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if @@ -1215,13 +1215,13 @@ .. method:: datetime.isoformat(sep='T', timespec='auto') Return a string representing the date and time in ISO 8601 format, - YYYY-MM-DDTHH:MM:SS.mmmmmm or, if :attr:`microsecond` is 0, + YYYY-MM-DDTHH:MM:SS.ffffff or, if :attr:`microsecond` is 0, YYYY-MM-DDTHH:MM:SS - If :meth:`utcoffset` does not return ``None``, a 6-character string is - appended, giving the UTC offset in (signed) hours and minutes: - YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM or, if :attr:`microsecond` is 0 - YYYY-MM-DDTHH:MM:SS+HH:MM + If :meth:`utcoffset` does not return ``None``, a string is + appended, giving the UTC offset: + YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] or, if :attr:`microsecond` + is 0 YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]. The optional argument *sep* (default ``'T'``) is a one-character separator, placed between the date and time portions of the result. For example, @@ -1245,7 +1245,7 @@ in HH:MM:SS format. - ``'milliseconds'``: Include full time, but truncate fractional second part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. + - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format. .. note:: @@ -1522,7 +1522,7 @@ Return a :class:`time` corresponding to a *time_string* in one of the formats emitted by :meth:`time.isoformat`. Specifically, this function supports - strings in the format(s) ``HH[:MM[:SS[.mmm[mmm]]]][+HH:MM[:SS[.ffffff]]]``. + strings in the format(s) ``HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]``. .. caution:: @@ -1548,10 +1548,10 @@ .. method:: time.isoformat(timespec='auto') - Return a string representing the time in ISO 8601 format, HH:MM:SS.mmmmmm or, if + Return a string representing the time in ISO 8601 format, HH:MM:SS.ffffff or, if :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a - 6-character string is appended, giving the UTC offset in (signed) hours and - minutes: HH:MM:SS.mmmmmm+HH:MM or, if self.microsecond is 0, HH:MM:SS+HH:MM + string is appended, giving the UTC offset: HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] + or, if self.microsecond is 0, HH:MM:SS+HH:MM[:SS[.ffffff]]. The optional argument *timespec* specifies the number of additional components of the time to include (the default is ``'auto'``). @@ -1565,7 +1565,7 @@ in HH:MM:SS format. - ``'milliseconds'``: Include full time, but truncate fractional second part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. + - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format. .. note:: @@ -2091,9 +2091,10 @@ | | number, zero-padded on the | 999999 | | | | left. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form | (empty), +0000, -0400, | \(6) | -| | ±HHMM[SS] (empty string if the | +1030 | | -| | object is naive). | | | +| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | +| | ±HHMM[SS[.ffffff]] (empty | -0400, +1030, | | +| | string if the object is | +063415, | | +| | naive). | -030712.345216 | | +-----------+--------------------------------+------------------------+-------+ | ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | | | if the object is naive). | | | @@ -2206,12 +2207,12 @@ ``%z`` :meth:`utcoffset` is transformed into a string of the form - ±HHMM[SS[.uuuuuu]], where HH is a 2-digit string giving the number of UTC - offset hours, and MM is a 2-digit string giving the number of UTC offset + ±HHMM[SS[.ffffff]], where HH is a 2-digit string giving the number of UTC + offset hours, MM is a 2-digit string giving the number of UTC offset minutes, SS is a 2-digit string giving the number of UTC offset - seconds and uuuuuu is a 2-digit string giving the number of UTC - offset microseconds. The uuuuuu part is omitted when the offset is a - whole number of minutes and both the uuuuuu and the SS parts are omitted + seconds and ffffff is a 6-digit string giving the number of UTC + offset microseconds. The ffffff part is omitted when the offset is a + whole number of seconds and both the ffffff and the SS part is omitted when the offset is a whole number of minutes. For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string ``'-0330'``. Binary files /tmp/tmptVuxje/EzpRrsVgCs/python3.7-3.7.0/Doc/library/depgraph-output.png and /tmp/tmptVuxje/_XOws4ynjS/python3.7-3.7.1/Doc/library/depgraph-output.png differ diff -Nru python3.7-3.7.0/Doc/library/dis.rst python3.7-3.7.1/Doc/library/dis.rst --- python3.7-3.7.0/Doc/library/dis.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/dis.rst 2018-10-20 06:04:19.000000000 +0000 @@ -149,7 +149,7 @@ .. function:: dis(x=None, *, file=None, depth=None) Disassemble the *x* object. *x* can denote either a module, a class, a - method, a function, a generator, an asynchronous generator, a couroutine, + method, a function, a generator, an asynchronous generator, a coroutine, a code object, a string of source code or a byte sequence of raw bytecode. For a module, it disassembles all functions. For a class, it disassembles all methods (including class and static methods). For a code object or @@ -1059,18 +1059,20 @@ .. opcode:: RAISE_VARARGS (argc) - Raises an exception. *argc* indicates the number of parameters to the raise + Raises an exception. *argc* indicates the number of arguments to the raise statement, ranging from 0 to 3. The handler will find the traceback as TOS2, the parameter as TOS1, and the exception as TOS. .. opcode:: CALL_FUNCTION (argc) - Calls a function. *argc* indicates the number of positional arguments. - The positional arguments are on the stack, with the right-most argument - on top. Below the arguments, the function object to call is on the stack. - Pops all function arguments, and the function itself off the stack, and - pushes the return value. + Calls a callable object with positional arguments. + *argc* indicates the number of positional arguments. + The top of the stack contains positional arguments, with the right-most + argument on top. Below the arguments is a callable object to call. + ``CALL_FUNCTION`` pops all arguments and the callable object off the stack, + calls the callable object with those arguments, and pushes the return value + returned by the callable object. .. versionchanged:: 3.6 This opcode is used only for calls with positional arguments. @@ -1078,31 +1080,36 @@ .. opcode:: CALL_FUNCTION_KW (argc) - Calls a function. *argc* indicates the number of arguments (positional - and keyword). The top element on the stack contains a tuple of keyword - argument names. Below the tuple, keyword arguments are on the stack, in - the order corresponding to the tuple. Below the keyword arguments, the - positional arguments are on the stack, with the right-most parameter on - top. Below the arguments, the function object to call is on the stack. - Pops all function arguments, and the function itself off the stack, and - pushes the return value. + Calls a callable object with positional (if any) and keyword arguments. + *argc* indicates the total number of positional and keyword arguments. + The top element on the stack contains a tuple of keyword argument names. + Below that are keyword arguments in the order corresponding to the tuple. + Below that are positional arguments, with the right-most parameter on + top. Below the arguments is a callable object to call. + ``CALL_FUNCTION_KW`` pops all arguments and the callable object off the stack, + calls the callable object with those arguments, and pushes the return value + returned by the callable object. .. versionchanged:: 3.6 Keyword arguments are packed in a tuple instead of a dictionary, - *argc* indicates the total number of arguments + *argc* indicates the total number of arguments. .. opcode:: CALL_FUNCTION_EX (flags) - Calls a function. The lowest bit of *flags* indicates whether the - var-keyword argument is placed at the top of the stack. Below the - var-keyword argument, the var-positional argument is on the stack. - Below the arguments, the function object to call is placed. - Pops all function arguments, and the function itself off the stack, and - pushes the return value. Note that this opcode pops at most three items - from the stack. Var-positional and var-keyword arguments are packed - by :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` and - :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. + Calls a callable object with variable set of positional and keyword + arguments. If the lowest bit of *flags* is set, the top of the stack + contains a mapping object containing additional keyword arguments. + Below that is an iterable object containing positional arguments and + a callable object to call. :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` can be used for merging multiple + mapping objects and iterables containing arguments. + Before the callable is called, the mapping object and iterable object + are each "unpacked" and their contents passed in as keyword and + positional arguments respectively. + ``CALL_FUNCTION_EX`` pops all arguments and the callable object off the stack, + calls the callable object with those arguments, and pushes the return value + returned by the callable object. .. versionadded:: 3.6 @@ -1134,7 +1141,8 @@ Pushes a new function object on the stack. From bottom to top, the consumed stack must consist of values if the argument carries a specified flag value - * ``0x01`` a tuple of default argument objects in positional order + * ``0x01`` a tuple of default values for positional-only and + positional-or-keyword parameters in positional order * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` an annotation dictionary * ``0x08`` a tuple containing cells for free variables, making a closure @@ -1217,7 +1225,7 @@ .. data:: hasconst - Sequence of bytecodes that have a constant parameter. + Sequence of bytecodes that access a constant. .. data:: hasfree diff -Nru python3.7-3.7.0/Doc/library/email.compat32-message.rst python3.7-3.7.1/Doc/library/email.compat32-message.rst --- python3.7-3.7.0/Doc/library/email.compat32-message.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/email.compat32-message.rst 2018-10-20 06:04:19.000000000 +0000 @@ -5,7 +5,7 @@ .. module:: email.message :synopsis: The base class representing email messages in a fashion - backward compatible with python3.2 + backward compatible with Python 3.2 The :class:`Message` class is very similar to the diff -Nru python3.7-3.7.0/Doc/library/email.errors.rst python3.7-3.7.1/Doc/library/email.errors.rst --- python3.7-3.7.0/Doc/library/email.errors.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/email.errors.rst 2018-10-20 06:04:19.000000000 +0000 @@ -108,3 +108,7 @@ * :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 encoded bytes, characters outside the base64 alphabet were encountered. The characters are ignored, but the resulting decoded bytes may be invalid. + +* :class:`InvalidBase64LengthDefect` -- When decoding a block of base64 encoded + bytes, the number of non-padding base64 characters was invalid (1 more than + a multiple of 4). The encoded block was kept as-is. diff -Nru python3.7-3.7.0/Doc/library/email.parser.rst python3.7-3.7.1/Doc/library/email.parser.rst --- python3.7-3.7.0/Doc/library/email.parser.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/email.parser.rst 2018-10-20 06:04:19.000000000 +0000 @@ -246,7 +246,7 @@ Removed the *strict* argument. Added the *policy* keyword. -.. function:: message_from_binary_file(fp, _class=None, *, +.. function:: message_from_binary_file(fp, _class=None, *, \ policy=policy.compat32) Return a message object structure tree from an open binary :term:`file diff -Nru python3.7-3.7.0/Doc/library/email.rst python3.7-3.7.1/Doc/library/email.rst --- python3.7-3.7.0/Doc/library/email.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/email.rst 2018-10-20 06:04:19.000000000 +0000 @@ -133,7 +133,7 @@ .. seealso:: Module :mod:`smtplib` - SMTP (Simple Mail Transport Protcol) client + SMTP (Simple Mail Transport Protocol) client Module :mod:`poplib` POP (Post Office Protocol) client diff -Nru python3.7-3.7.0/Doc/library/enum.rst python3.7-3.7.1/Doc/library/enum.rst --- python3.7-3.7.0/Doc/library/enum.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/enum.rst 2018-10-20 06:04:19.000000000 +0000 @@ -4,10 +4,10 @@ .. module:: enum :synopsis: Implementation of an enumeration class. -.. :moduleauthor:: Ethan Furman -.. :sectionauthor:: Barry Warsaw , -.. :sectionauthor:: Eli Bendersky , -.. :sectionauthor:: Ethan Furman +.. moduleauthor:: Ethan Furman +.. sectionauthor:: Barry Warsaw +.. sectionauthor:: Eli Bendersky +.. sectionauthor:: Ethan Furman .. versionadded:: 3.4 @@ -387,10 +387,17 @@ methods. See `Planet`_ for an example. -Restricted subclassing of enumerations --------------------------------------- +Restricted Enum subclassing +--------------------------- -Subclassing an enumeration is allowed only if the enumeration does not define +A new :class:`Enum` class must have one base Enum class, up to one concrete +data type, and as many :class:`object`-based mixin classes as needed. The +order of these base classes is:: + + def EnumName([mix-in, ...,] [data-type,] base-enum): + pass + +Also, subclassing an enumeration is allowed only if the enumeration does not define any members. So this is forbidden:: >>> class MoreColor(Color): diff -Nru python3.7-3.7.0/Doc/library/exceptions.rst python3.7-3.7.1/Doc/library/exceptions.rst --- python3.7-3.7.0/Doc/library/exceptions.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/exceptions.rst 2018-10-20 06:04:19.000000000 +0000 @@ -525,7 +525,7 @@ .. exception:: ValueError - Raised when a built-in operation or function receives an argument that has the + Raised when an operation or function receives an argument that has the right type but an inappropriate value, and the situation is not described by a more precise exception such as :exc:`IndexError`. diff -Nru python3.7-3.7.0/Doc/library/functions.rst python3.7-3.7.1/Doc/library/functions.rst --- python3.7-3.7.0/Doc/library/functions.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/functions.rst 2018-10-20 06:04:19.000000000 +0000 @@ -98,7 +98,7 @@ >>> f'{14:#b}', f'{14:b}' ('0b1110', '1110') - See also :func:`format` for more information. + See also :func:`format` for more information. .. class:: bool([x]) @@ -112,6 +112,8 @@ .. index:: pair: Boolean; type + .. versionchanged:: 3.7 + *x* is now a positional-only parameter. .. function:: breakpoint(*args, **kws) @@ -240,8 +242,8 @@ interactive statement (in the latter case, expression statements that evaluate to something other than ``None`` will be printed). - The optional arguments *flags* and *dont_inherit* control which future - statements (see :pep:`236`) affect the compilation of *source*. If neither + The optional arguments *flags* and *dont_inherit* control which :ref:`future + statements ` affect the compilation of *source*. If neither is present (or both are zero) the code is compiled with those future statements that are in effect in the code that is calling :func:`compile`. If the *flags* argument is given and *dont_inherit* is not (or is zero) then the @@ -434,8 +436,10 @@ The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* dictionaries as global and local namespace. If the *globals* dictionary is - present and lacks '__builtins__', the current globals are copied into *globals* - before *expression* is parsed. This means that *expression* normally has full + present and does not contain a value for the key ``__builtins__``, a + reference to the dictionary of the built-in module :mod:`builtins` is + inserted under that key before *expression* is parsed. + This means that *expression* normally has full access to the standard :mod:`builtins` module and restricted environments are propagated. If the *locals* dictionary is omitted it defaults to the *globals* dictionary. If both dictionaries are omitted, the expression is executed in the @@ -575,6 +579,9 @@ .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. + .. versionchanged:: 3.7 + *x* is now a positional-only parameter. + .. index:: single: __format__ @@ -646,11 +653,11 @@ dictionary lookup. Numeric values that compare equal have the same hash value (even if they are of different types, as is the case for 1 and 1.0). - .. note:: + .. note:: - For objects with custom :meth:`__hash__` methods, note that :func:`hash` - truncates the return value based on the bit width of the host machine. - See :meth:`__hash__` for details. + For objects with custom :meth:`__hash__` methods, note that :func:`hash` + truncates the return value based on the bit width of the host machine. + See :meth:`__hash__` for details. .. function:: help([object]) @@ -726,7 +733,7 @@ to provide elaborate line editing and history features. -.. class:: int(x=0) +.. class:: int([x]) int(x, base=10) Return an integer object constructed from a number or string *x*, or return @@ -760,6 +767,9 @@ .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. + .. versionchanged:: 3.7 + *x* is now a positional-only parameter. + .. function:: isinstance(object, classinfo) @@ -971,6 +981,11 @@ encoding. (For reading and writing raw bytes use binary mode and leave *encoding* unspecified.) The available modes are: + .. _filemodes: + + .. index:: + pair: file; modes + ========= =============================================================== Character Meaning ========= =============================================================== diff -Nru python3.7-3.7.0/Doc/library/functools.rst python3.7-3.7.1/Doc/library/functools.rst --- python3.7-3.7.0/Doc/library/functools.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/functools.rst 2018-10-20 06:04:19.000000000 +0000 @@ -52,6 +52,11 @@ Since a dictionary is used to cache results, the positional and keyword arguments to the function must be hashable. + Distinct argument patterns may be considered to be distinct calls with + separate cache entries. For example, `f(a=1, b=2)` and `f(b=2, a=1)` + differ in their keyword argument order and may have two separate cache + entries. + If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can grow without bound. The LRU feature performs best when *maxsize* is a power-of-two. diff -Nru python3.7-3.7.0/Doc/library/hashlib.rst python3.7-3.7.1/Doc/library/hashlib.rst --- python3.7-3.7.0/Doc/library/hashlib.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/hashlib.rst 2018-10-20 06:04:19.000000000 +0000 @@ -101,7 +101,7 @@ .. function:: new(name[, data]) - Is a generic constructor that takes the string name of the desired + Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the above listed hashes as well as any other algorithms that your OpenSSL library may offer. The named constructors are much faster than :func:`new` @@ -162,10 +162,10 @@ A hash object has the following methods: -.. method:: hash.update(arg) +.. method:: hash.update(data) - Update the hash object with the object *arg*, which must be interpretable as - a buffer of bytes. Repeated calls are equivalent to a single call with the + Update the hash object with the :term:`bytes-like object`. + Repeated calls are equivalent to a single call with the concatenation of all the arguments: ``m.update(a); m.update(b)`` is equivalent to ``m.update(a+b)``. @@ -206,7 +206,7 @@ .. method:: shake.digest(length) Return the digest of the data passed to the :meth:`update` method so far. - This is a bytes object of size ``length`` which may contain bytes in + This is a bytes object of size *length* which may contain bytes in the whole range from 0 to 255. @@ -262,9 +262,10 @@ The function provides scrypt password-based key derivation function as defined in :rfc:`7914`. - *password* and *salt* must be bytes-like objects. Applications and - libraries should limit *password* to a sensible length (e.g. 1024). *salt* - should be about 16 or more bytes from a proper source, e.g. :func:`os.urandom`. + *password* and *salt* must be :term:`bytes-like objects + `. Applications and libraries should limit *password* + to a sensible length (e.g. 1024). *salt* should be about 16 or more + bytes from a proper source, e.g. :func:`os.urandom`. *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). @@ -305,11 +306,11 @@ New hash objects are created by calling constructor functions: -.. function:: blake2b(data=b'', digest_size=64, key=b'', salt=b'', \ +.. function:: blake2b(data=b'', *, digest_size=64, key=b'', salt=b'', \ person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ node_depth=0, inner_size=0, last_node=False) -.. function:: blake2s(data=b'', digest_size=32, key=b'', salt=b'', \ +.. function:: blake2s(data=b'', *, digest_size=32, key=b'', salt=b'', \ person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ node_depth=0, inner_size=0, last_node=False) @@ -317,8 +318,8 @@ These functions return the corresponding hash objects for calculating BLAKE2b or BLAKE2s. They optionally take these general parameters: -* *data*: initial chunk of data to hash, which must be interpretable as buffer - of bytes. +* *data*: initial chunk of data to hash, which must be + :term:`bytes-like object`. It can be passed only as positional argument. * *digest_size*: size of output digest in bytes. @@ -427,7 +428,7 @@ As a shortcut, you can pass the first chunk of data to update directly to the -constructor as the first argument (or as *data* keyword argument): +constructor as the positional argument: >>> from hashlib import blake2b >>> blake2b(b'Hello world').hexdigest() @@ -546,7 +547,7 @@ preparer, generates all or part of a message to be signed by a second party, the message signer. If the message preparer is able to find cryptographic hash function collisions (i.e., two messages producing the - same hash value), then she might prepare meaningful versions of the message + same hash value), then they might prepare meaningful versions of the message that would produce the same hash value and digital signature, but with different results (e.g., transferring $1,000,000 to an account, rather than $10). Cryptographic hash functions have been designed with collision diff -Nru python3.7-3.7.0/Doc/library/html.rst python3.7-3.7.1/Doc/library/html.rst --- python3.7-3.7.0/Doc/library/html.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/html.rst 2018-10-20 06:04:19.000000000 +0000 @@ -24,7 +24,7 @@ .. function:: unescape(s) Convert all named and numeric character references (e.g. ``>``, - ``>``, ``&x3e;``) in the string *s* to the corresponding unicode + ``>``, ``>``) in the string *s* to the corresponding Unicode characters. This function uses the rules defined by the HTML 5 standard for both valid and invalid character references, and the :data:`list of HTML 5 named character references `. diff -Nru python3.7-3.7.0/Doc/library/idle.rst python3.7-3.7.1/Doc/library/idle.rst --- python3.7-3.7.0/Doc/library/idle.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/idle.rst 2018-10-20 06:04:19.000000000 +0000 @@ -303,7 +303,7 @@ and open docs.python.org showing the latest Python documentation. Turtle Demo - Run the turtledemo module with example python code and turtle drawings. + Run the turtledemo module with example Python code and turtle drawings. Additional help sources may be added here with the Configure IDLE dialog under the General tab. diff -Nru python3.7-3.7.0/Doc/library/imaplib.rst python3.7-3.7.1/Doc/library/imaplib.rst --- python3.7-3.7.0/Doc/library/imaplib.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/imaplib.rst 2018-10-20 06:04:19.000000000 +0000 @@ -517,7 +517,7 @@ create such tags. Although it is an RFC violation and IMAP clients and servers are supposed to be strict, imaplib nonetheless continues to allow such tags to be created for backward compatibility reasons, and as of - python 3.6, handles them if they are sent from the server, since this + Python 3.6, handles them if they are sent from the server, since this improves real-world compatibility. .. method:: IMAP4.subscribe(mailbox) diff -Nru python3.7-3.7.0/Doc/library/importlib.rst python3.7-3.7.1/Doc/library/importlib.rst --- python3.7-3.7.0/Doc/library/importlib.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/importlib.rst 2018-10-20 06:04:19.000000000 +0000 @@ -249,7 +249,7 @@ .. abstractmethod:: find_module(fullname, path=None) - An abstact method for finding a :term:`loader` for the specified + An abstract method for finding a :term:`loader` for the specified module. Originally specified in :pep:`302`, this method was meant for use in :data:`sys.meta_path` and in the path-based import subsystem. @@ -770,7 +770,7 @@ Concrete implementation of :meth:`Loader.exec_module`. - .. versionadded:: 3.4 + .. versionadded:: 3.4 .. method:: load_module(fullname) diff -Nru python3.7-3.7.0/Doc/library/index.rst python3.7-3.7.1/Doc/library/index.rst --- python3.7-3.7.0/Doc/library/index.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/index.rst 2018-10-20 06:04:19.000000000 +0000 @@ -32,10 +32,11 @@ packages and entire application development frameworks), available from the `Python Package Index `_. - +.. We don't use :numbered: option for the TOC below as it enforces + numbered sections for the entire stdlib docs. If desired, + :numbered: can be enabled on a per-module basis. .. toctree:: :maxdepth: 2 - :numbered: intro.rst functions.rst diff -Nru python3.7-3.7.0/Doc/library/inspect.rst python3.7-3.7.1/Doc/library/inspect.rst --- python3.7-3.7.0/Doc/library/inspect.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/inspect.rst 2018-10-20 06:04:19.000000000 +0000 @@ -556,7 +556,7 @@ >>> sig.parameters['b'].annotation - Accepts a wide range of python callables, from plain functions and classes to + Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. Raises :exc:`ValueError` if no signature can be provided, and diff -Nru python3.7-3.7.0/Doc/library/io.rst python3.7-3.7.1/Doc/library/io.rst --- python3.7-3.7.0/Doc/library/io.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/io.rst 2018-10-20 06:04:19.000000000 +0000 @@ -799,7 +799,7 @@ .. versionadded:: 3.1 - .. method:: read(size) + .. method:: read(size=-1) Read and return at most *size* characters from the stream as a single :class:`str`. If *size* is negative or ``None``, reads until EOF. diff -Nru python3.7-3.7.0/Doc/library/ipc.rst python3.7-3.7.1/Doc/library/ipc.rst --- python3.7-3.7.0/Doc/library/ipc.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/ipc.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,11 +1,11 @@ .. _ipc: ***************************************** -Interprocess Communication and Networking +Networking and Interprocess Communication ***************************************** -The modules described in this chapter provide mechanisms for different processes -to communicate. +The modules described in this chapter provide mechanisms for +networking and inter-processes communication. Some modules only work for two processes that are on the same machine, e.g. :mod:`signal` and :mod:`mmap`. Other modules support networking protocols @@ -15,12 +15,13 @@ .. toctree:: + :maxdepth: 1 + asyncio.rst socket.rst ssl.rst select.rst selectors.rst - asyncio.rst asyncore.rst asynchat.rst signal.rst diff -Nru python3.7-3.7.0/Doc/library/json.rst python3.7-3.7.1/Doc/library/json.rst --- python3.7-3.7.0/Doc/library/json.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/json.rst 2018-10-20 06:04:19.000000000 +0000 @@ -188,6 +188,11 @@ .. versionchanged:: 3.6 All optional parameters are now :ref:`keyword-only `. + .. note:: + + Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, + so trying to serialize multiple objects with repeated calls to + :func:`dump` using the same *fp* will result in an invalid JSON file. .. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ check_circular=True, allow_nan=True, cls=None, \ @@ -200,12 +205,6 @@ .. note:: - Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, - so trying to serialize multiple objects with repeated calls to - :func:`dump` using the same *fp* will result in an invalid JSON file. - - .. note:: - Keys in key/value pairs of JSON are always of the type :class:`str`. When a dictionary is converted into JSON, all the keys of the dictionary are coerced to strings. As a result of this, if a dictionary is converted diff -Nru python3.7-3.7.0/Doc/library/logging.handlers.rst python3.7-3.7.1/Doc/library/logging.handlers.rst --- python3.7-3.7.0/Doc/library/logging.handlers.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/logging.handlers.rst 2018-10-20 06:04:19.000000000 +0000 @@ -973,9 +973,9 @@ Prepares a record for queuing. The object returned by this method is enqueued. - The base implementation formats the record to merge the message - and arguments, and removes unpickleable items from the record - in-place. + The base implementation formats the record to merge the message, + arguments, and exception information, if present. It also + removes unpickleable items from the record in-place. You might want to override this method if you want to convert the record to a dict or JSON string, or send a modified copy diff -Nru python3.7-3.7.0/Doc/library/logging.rst python3.7-3.7.1/Doc/library/logging.rst --- python3.7-3.7.0/Doc/library/logging.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/logging.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1129,52 +1129,54 @@ +--------------+---------------------------------------------+ | Format | Description | +==============+=============================================+ - | ``filename`` | Specifies that a FileHandler be created, | + | *filename* | Specifies that a FileHandler be created, | | | using the specified filename, rather than a | | | StreamHandler. | +--------------+---------------------------------------------+ - | ``filemode`` | Specifies the mode to open the file, if | - | | filename is specified (if filemode is | - | | unspecified, it defaults to 'a'). | + | *filemode* | If *filename* is specified, open the file | + | | in this :ref:`mode `. Defaults | + | | to ``'a'``. | +--------------+---------------------------------------------+ - | ``format`` | Use the specified format string for the | + | *format* | Use the specified format string for the | | | handler. | +--------------+---------------------------------------------+ - | ``datefmt`` | Use the specified date/time format. | + | *datefmt* | Use the specified date/time format, as | + | | accepted by :func:`time.strftime`. | +--------------+---------------------------------------------+ - | ``style`` | If ``format`` is specified, use this style | - | | for the format string. One of '%', '{' or | - | | '$' for %-formatting, :meth:`str.format` or | - | | :class:`string.Template` respectively, and | - | | defaulting to '%' if not specified. | + | *style* | If *format* is specified, use this style | + | | for the format string. One of ``'%'``, | + | | ``'{'`` or ``'$'`` for :ref:`printf-style | + | | `, | + | | :meth:`str.format` or | + | | :class:`string.Template` respectively. | + | | Defaults to ``'%'``. | +--------------+---------------------------------------------+ - | ``level`` | Set the root logger level to the specified | - | | level. | + | *level* | Set the root logger level to the specified | + | | :ref:`level `. | +--------------+---------------------------------------------+ - | ``stream`` | Use the specified stream to initialize the | + | *stream* | Use the specified stream to initialize the | | | StreamHandler. Note that this argument is | - | | incompatible with 'filename' - if both are | - | | present, a ``ValueError`` is raised. | + | | incompatible with *filename* - if both | + | | are present, a ``ValueError`` is raised. | +--------------+---------------------------------------------+ - | ``handlers`` | If specified, this should be an iterable of | + | *handlers* | If specified, this should be an iterable of | | | already created handlers to add to the root | | | logger. Any handlers which don't already | | | have a formatter set will be assigned the | | | default formatter created in this function. | | | Note that this argument is incompatible | - | | with 'filename' or 'stream' - if both are | - | | present, a ``ValueError`` is raised. | + | | with *filename* or *stream* - if both | + | | are present, a ``ValueError`` is raised. | +--------------+---------------------------------------------+ .. versionchanged:: 3.2 - The ``style`` argument was added. + The *style* argument was added. .. versionchanged:: 3.3 - The ``handlers`` argument was added. Additional checks were added to + The *handlers* argument was added. Additional checks were added to catch situations where incompatible arguments are specified (e.g. - ``handlers`` together with ``stream`` or ``filename``, or ``stream`` - together with ``filename``). - + *handlers* together with *stream* or *filename*, or *stream* + together with *filename*). .. function:: shutdown() diff -Nru python3.7-3.7.0/Doc/library/optparse.rst python3.7-3.7.1/Doc/library/optparse.rst --- python3.7-3.7.0/Doc/library/optparse.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/optparse.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1677,7 +1677,7 @@ problems with the option or its argument(s). :mod:`optparse` catches this and terminates the program, printing the error message you supply to stderr. Your message should be clear, concise, accurate, and mention the option at fault. -Otherwise, the user will have a hard time figuring out what he did wrong. +Otherwise, the user will have a hard time figuring out what they did wrong. .. _optparse-callback-example-1: diff -Nru python3.7-3.7.0/Doc/library/pathlib.rst python3.7-3.7.1/Doc/library/pathlib.rst --- python3.7-3.7.0/Doc/library/pathlib.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/pathlib.rst 2018-10-20 06:04:19.000000000 +0000 @@ -559,7 +559,8 @@ .. method:: PurePath.with_suffix(suffix) Return a new path with the :attr:`suffix` changed. If the original path - doesn't have a suffix, the new *suffix* is appended instead:: + doesn't have a suffix, the new *suffix* is appended instead. If the + *suffix* is an empty string, the original suffix is removed:: >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') >>> p.with_suffix('.bz2') @@ -567,6 +568,9 @@ >>> p = PureWindowsPath('README') >>> p.with_suffix('.txt') PureWindowsPath('README.txt') + >>> p = PureWindowsPath('README.txt') + >>> p.with_suffix('') + PureWindowsPath('README') .. _concrete-paths: @@ -911,7 +915,8 @@ >>> p.read_text() 'Text file contents' - The optional parameters have the same meaning as in :func:`open`. + The file is opened and then closed. The optional parameters have the same + meaning as in :func:`open`. .. versionadded:: 3.5 @@ -1080,23 +1085,30 @@ overlapping use-cases, their semantics differ enough to warrant not considering them equivalent. -============================ ============================== -os and os.path pathlib -============================ ============================== -:func:`os.path.abspath` :meth:`Path.resolve` -:func:`os.getcwd` :func:`Path.cwd` -:func:`os.path.exists` :meth:`Path.exists` -:func:`os.path.expanduser` :meth:`Path.expanduser` and - :meth:`Path.home` -:func:`os.path.isdir` :meth:`Path.is_dir` -:func:`os.path.isfile` :meth:`Path.is_file` -:func:`os.path.islink` :meth:`Path.is_symlink` -:func:`os.stat` :meth:`Path.stat`, - :meth:`Path.owner`, - :meth:`Path.group` -:func:`os.path.isabs` :meth:`PurePath.is_absolute` -:func:`os.path.join` :func:`PurePath.joinpath` -:func:`os.path.basename` :data:`PurePath.name` -:func:`os.path.dirname` :data:`PurePath.parent` -:func:`os.path.splitext` :data:`PurePath.suffix` -============================ ============================== +==================================== ============================== +os and os.path pathlib +==================================== ============================== +:func:`os.path.abspath` :meth:`Path.resolve` +:func:`os.chmod` :meth:`Path.chmod` +:func:`os.mkdir` :meth:`Path.mkdir` +:func:`os.rename` :meth:`Path.rename` +:func:`os.replace` :meth:`Path.replace` +:func:`os.rmdir` :meth:`Path.rmdir` +:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` +:func:`os.getcwd` :func:`Path.cwd` +:func:`os.path.exists` :meth:`Path.exists` +:func:`os.path.expanduser` :meth:`Path.expanduser` and + :meth:`Path.home` +:func:`os.path.isdir` :meth:`Path.is_dir` +:func:`os.path.isfile` :meth:`Path.is_file` +:func:`os.path.islink` :meth:`Path.is_symlink` +:func:`os.stat` :meth:`Path.stat`, + :meth:`Path.owner`, + :meth:`Path.group` +:func:`os.path.isabs` :meth:`PurePath.is_absolute` +:func:`os.path.join` :func:`PurePath.joinpath` +:func:`os.path.basename` :data:`PurePath.name` +:func:`os.path.dirname` :data:`PurePath.parent` +:func:`os.path.samefile` :meth:`Path.samefile` +:func:`os.path.splitext` :data:`PurePath.suffix` +==================================== ============================== diff -Nru python3.7-3.7.0/Doc/library/pdb.rst python3.7-3.7.1/Doc/library/pdb.rst --- python3.7-3.7.0/Doc/library/pdb.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/pdb.rst 2018-10-20 06:04:19.000000000 +0000 @@ -344,7 +344,7 @@ Specifying any command resuming execution (currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`, :pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) - terminates the command :pdbcmd:`list` (as if + terminates the command list (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint—which could have its own command list, leading to ambiguities about diff -Nru python3.7-3.7.0/Doc/library/platform.rst python3.7-3.7.1/Doc/library/platform.rst --- python3.7-3.7.0/Doc/library/platform.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/platform.rst 2018-10-20 06:04:19.000000000 +0000 @@ -270,7 +270,7 @@ .. deprecated-removed:: 3.5 3.8 See alternative like the `distro `_ package. -.. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=2048) +.. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=16384) Tries to determine the libc version against which the file executable (defaults to the Python interpreter) is linked. Returns a tuple of strings ``(lib, diff -Nru python3.7-3.7.0/Doc/library/profile.rst python3.7-3.7.1/Doc/library/profile.rst --- python3.7-3.7.0/Doc/library/profile.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/profile.rst 2018-10-20 06:04:19.000000000 +0000 @@ -297,6 +297,11 @@ Profile ``func(*args, **kwargs)`` +Note that profiling will only work if the called command/function actually +returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call +during the called command/function execution) no profiling results will be +printed. + .. _profile-stats: The :class:`Stats` Class diff -Nru python3.7-3.7.0/Doc/library/pyclbr.rst python3.7-3.7.1/Doc/library/pyclbr.rst --- python3.7-3.7.0/Doc/library/pyclbr.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/pyclbr.rst 2018-10-20 06:04:19.000000000 +0000 @@ -11,9 +11,9 @@ -------------- The :mod:`pyclbr` module provides limited information about the -functions, classes, and methods defined in a python-coded module. The +functions, classes, and methods defined in a Python-coded module. The information is sufficient to implement a module browser. The -information is extracted from the python source code rather than by +information is extracted from the Python source code rather than by importing the module, so this module is safe to use with untrusted code. This restriction makes it impossible to use this module with modules not implemented in Python, including all standard and optional extension diff -Nru python3.7-3.7.0/Doc/library/re.rst python3.7-3.7.1/Doc/library/re.rst --- python3.7-3.7.0/Doc/library/re.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/re.rst 2018-10-20 06:04:19.000000000 +0000 @@ -204,7 +204,7 @@ Standard #18`_ might be added in the future. This would change the syntax, so to facilitate this change a :exc:`FutureWarning` will be raised in ambiguous cases for the time being. - That include sets starting with a literal ``'['`` or containing literal + That includes sets starting with a literal ``'['`` or containing literal character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To avoid a warning escape them with a backslash. @@ -1447,8 +1447,8 @@ ^^^^^^^^^^^^^^^^^^^ :func:`findall` matches *all* occurrences of a pattern, not just the first -one as :func:`search` does. For example, if one was a writer and wanted to -find all of the adverbs in some text, he or she might use :func:`findall` in +one as :func:`search` does. For example, if a writer wanted to +find all of the adverbs in some text, they might use :func:`findall` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." @@ -1462,8 +1462,8 @@ If one wants more information about all matches of a pattern than the matched text, :func:`finditer` is useful as it provides :ref:`match objects ` instead of strings. Continuing with the previous example, if -one was a writer who wanted to find all of the adverbs *and their positions* in -some text, he or she would use :func:`finditer` in the following manner:: +a writer wanted to find all of the adverbs *and their positions* in +some text, they would use :func:`finditer` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." >>> for m in re.finditer(r"\w+ly", text): diff -Nru python3.7-3.7.0/Doc/library/select.rst python3.7-3.7.1/Doc/library/select.rst --- python3.7-3.7.0/Doc/library/select.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/select.rst 2018-10-20 06:04:19.000000000 +0000 @@ -57,7 +57,16 @@ (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O - events. *sizehint* and *flags* are deprecated and completely ignored. + events. + + *sizehint* informs epoll about the expected number of events to be + registered. It must be positive, or `-1` to use the default. It is only + used on older systems where :c:func:`epoll_create1` is not available; + otherwise it has no effect (though its value is still checked). + + *flags* is deprecated and completely ignored. However, when supplied, its + value must be ``0`` or ``select.EPOLL_CLOEXEC``, otherwise ``OSError`` is + raised. See the :ref:`epoll-objects` section below for the methods supported by epolling objects. diff -Nru python3.7-3.7.0/Doc/library/signal.rst python3.7-3.7.1/Doc/library/signal.rst --- python3.7-3.7.0/Doc/library/signal.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/signal.rst 2018-10-20 06:04:19.000000000 +0000 @@ -485,3 +485,37 @@ signal.alarm(0) # Disable the alarm +Note on SIGPIPE +--------------- + +Piping output of your program to tools like :manpage:`head(1)` will +cause a :const:`SIGPIPE` signal to be sent to your process when the receiver +of its standard output closes early. This results in an exception +like :code:`BrokenPipeError: [Errno 32] Broken pipe`. To handle this +case, wrap your entry point to catch this exception as follows:: + + import os + import sys + + def main(): + try: + # simulate large output (your code replaces this loop) + for x in range(10000): + print("y") + # flush output here to force SIGPIPE to be triggered + # while inside this try block. + sys.stdout.flush() + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE + + if __name__ == '__main__': + main() + +Do not set :const:`SIGPIPE`'s disposition to :const:`SIG_DFL` +in order to avoid :exc:`BrokenPipeError`. Doing that would cause +your program to exit unexpectedly also whenever any socket connection +is interrupted while your program is still writing to it. diff -Nru python3.7-3.7.0/Doc/library/site.rst python3.7-3.7.1/Doc/library/site.rst --- python3.7-3.7.0/Doc/library/site.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/site.rst 2018-10-20 06:04:19.000000000 +0000 @@ -245,7 +245,7 @@ If both options are given, user base and user site will be printed (always in this order), separated by :data:`os.pathsep`. -If any option is given, the script will exit with one of these values: ``O`` if +If any option is given, the script will exit with one of these values: ``0`` if the user site-packages directory is enabled, ``1`` if it was disabled by the user, ``2`` if it is disabled for security reasons or by an administrator, and a value greater than 2 if there is an error. diff -Nru python3.7-3.7.0/Doc/library/smtplib.rst python3.7-3.7.1/Doc/library/smtplib.rst --- python3.7-3.7.0/Doc/library/smtplib.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/smtplib.rst 2018-10-20 06:04:19.000000000 +0000 @@ -271,7 +271,7 @@ .. method:: SMTP.ehlo_or_helo_if_needed() - This method call :meth:`ehlo` and or :meth:`helo` if there has been no + This method calls :meth:`ehlo` and/or :meth:`helo` if there has been no previous ``EHLO`` or ``HELO`` command this session. It tries ESMTP ``EHLO`` first. @@ -346,7 +346,7 @@ If optional keyword argument *initial_response_ok* is true, ``authobject()`` will be called first with no argument. It can return the - :rfc:`4954` "initial response" bytes which will be encoded and sent with + :rfc:`4954` "initial response" ASCII ``str`` which will be encoded and sent with the ``AUTH`` command as below. If the ``authobject()`` does not support an initial response (e.g. because it requires a challenge), it should return ``None`` when called with ``challenge=None``. If *initial_response_ok* is @@ -355,7 +355,7 @@ If the initial response check returns ``None``, or if *initial_response_ok* is false, ``authobject()`` will be called to process the server's challenge response; the *challenge* argument it is passed will be a ``bytes``. It - should return ``bytes`` *data* that will be base64 encoded and sent to the + should return ASCII ``str`` *data* that will be base64 encoded and sent to the server. The ``SMTP`` class provides ``authobjects`` for the ``CRAM-MD5``, ``PLAIN``, @@ -379,16 +379,23 @@ commands that follow will be encrypted. You should then call :meth:`ehlo` again. - If *keyfile* and *certfile* are provided, these are passed to the :mod:`socket` - module's :func:`ssl` function. + If *keyfile* and *certfile* are provided, they are used to create an + :class:`ssl.SSLContext`. - Optional *context* parameter is a :class:`ssl.SSLContext` object; This is + Optional *context* parameter is an :class:`ssl.SSLContext` object; This is an alternative to using a keyfile and a certfile and if specified both *keyfile* and *certfile* should be ``None``. If there has been no previous ``EHLO`` or ``HELO`` command this session, this method tries ESMTP ``EHLO`` first. + .. deprecated:: 3.6 + + *keyfile* and *certfile* are deprecated in favor of *context*. + Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let + :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. + :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. @@ -412,7 +419,7 @@ :exc:`SMTPException`. -.. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[]) +.. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=(), rcpt_options=()) Send mail. The required arguments are an :rfc:`822` from-address string, a list of :rfc:`822` to-address strings (a bare string will be treated as a list with 1 @@ -484,7 +491,7 @@ .. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \ - mail_options=[], rcpt_options=[]) + mail_options=(), rcpt_options=()) This is a convenience method for calling :meth:`sendmail` with the message represented by an :class:`email.message.Message` object. The arguments have diff -Nru python3.7-3.7.0/Doc/library/socket.rst python3.7-3.7.1/Doc/library/socket.rst --- python3.7-3.7.0/Doc/library/socket.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/socket.rst 2018-10-20 06:04:19.000000000 +0000 @@ -70,6 +70,13 @@ notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``, and *port* is an integer. + - For IPv4 addresses, two special forms are accepted instead of a host + address: ``''`` represents :const:`INADDR_ANY`, which is used to bind to all + interfaces, and the string ``''`` represents + :const:`INADDR_BROADCAST`. This behavior is not compatible with IPv6, + therefore, you may want to avoid these if you intend to support IPv6 with your + Python programs. + - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo`` and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For @@ -166,16 +173,25 @@ .. versionadded:: 3.7 -- Certain other address families (:const:`AF_PACKET`, :const:`AF_CAN`) - support specific representations. - - .. XXX document them! - -For IPv4 addresses, two special forms are accepted instead of a host address: -the empty string represents :const:`INADDR_ANY`, and the string -``''`` represents :const:`INADDR_BROADCAST`. This behavior is not -compatible with IPv6, therefore, you may want to avoid these if you intend -to support IPv6 with your Python programs. +- :const:`AF_PACKET` is a low-level interface directly to network devices. + The packets are represented by the tuple + ``(ifname, proto[, pkttype[, hatype[, addr]]])`` where: + + - *ifname* - String specifying the device name. + - *proto* - An in network-byte-order integer specifying the Ethernet + protocol number. + - *pkttype* - Optional integer specifying the packet type: + + - ``PACKET_HOST`` (the default) - Packet addressed to the local host. + - ``PACKET_BROADCAST`` - Physical-layer broadcast packet. + - ``PACKET_MULTIHOST`` - Packet sent to a physical-layer multicast address. + - ``PACKET_OTHERHOST`` - Packet to some other host that has been caught by + a device driver in promiscuous mode. + - ``PACKET_OUTGOING`` - Packet originating from the local host that is + looped back to a packet socket. + - *hatype* - Optional integer specifying the ARP hardware address type. + - *addr* - Optional bytes-like object specifying the hardware physical + address, whose interpretation depends on the device. If you use a hostname in the *host* portion of IPv4/v6 socket address, the program may show a nondeterministic behavior, as Python uses the first address @@ -375,6 +391,16 @@ .. versionadded:: 3.7 +.. data:: AF_PACKET + PF_PACKET + PACKET_* + + Many constants of these forms, documented in the Linux documentation, are + also defined in the socket module. + + Availability: Linux >= 2.2. + + .. data:: AF_RDS PF_RDS SOL_RDS @@ -468,12 +494,12 @@ Create a new socket using the given address family, socket type and protocol number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN` or :const:`AF_RDS`. The - socket type should be :const:`SOCK_STREAM` (the default), - :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` - constants. The protocol number is usually zero and may be omitted or in the - case where the address family is :const:`AF_CAN` the protocol should be one - of :const:`CAN_RAW`, :const:`CAN_BCM` or :const:`CAN_ISOTP` + :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, + or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the + default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other + ``SOCK_`` constants. The protocol number is usually zero and may be omitted + or in the case where the address family is :const:`AF_CAN` the protocol + should be one of :const:`CAN_RAW`, :const:`CAN_BCM` or :const:`CAN_ISOTP`. If *fileno* is specified, the values for *family*, *type*, and *proto* are auto-detected from the specified file descriptor. Auto-detection can be diff -Nru python3.7-3.7.0/Doc/library/sqlite3.rst python3.7-3.7.1/Doc/library/sqlite3.rst --- python3.7-3.7.0/Doc/library/sqlite3.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/sqlite3.rst 2018-10-20 06:04:19.000000000 +0000 @@ -236,8 +236,8 @@ Registers a callable to convert a bytestring from the database into a custom Python type. The callable will be invoked for all database values that are of the type *typename*. Confer the parameter *detect_types* of the :func:`connect` - function for how the type detection works. Note that the case of *typename* and - the name of the type in your query must match! + function for how the type detection works. Note that *typename* and the name of + the type in your query are matched in case-insensitive manner. .. function:: register_adapter(type, callable) @@ -281,7 +281,7 @@ .. attribute:: isolation_level - Get or set the current isolation level. :const:`None` for autocommit mode or + Get or set the current default isolation level. :const:`None` for autocommit mode or one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section :ref:`sqlite3-controlling-transactions` for a more detailed explanation. @@ -821,6 +821,20 @@ exists, syntax error in the SQL statement, wrong number of parameters specified, etc. It is a subclass of :exc:`DatabaseError`. +.. exception:: OperationalError + + Exception raised for errors that are related to the database's operation + and not necessarily under the control of the programmer, e.g. an unexpected + disconnect occurs, the data source name is not found, a transaction could + not be processed, etc. It is a subclass of :exc:`DatabaseError`. + +.. exception:: NotSupportedError + + Exception raised in case a method or database API was used which is not + supported by the database, e.g. calling the :meth:`~Connection.rollback` + method on a connection that does not support transaction or has + transactions turned off. It is a subclass of :exc:`DatabaseError`. + .. _sqlite3-types: @@ -989,22 +1003,30 @@ Controlling Transactions ------------------------ -By default, the :mod:`sqlite3` module opens transactions implicitly before a -Data Modification Language (DML) statement (i.e. -``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). +The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, +but the Python :mod:`sqlite3` module by default does not. -You can control which kind of ``BEGIN`` statements sqlite3 implicitly executes -(or none at all) via the *isolation_level* parameter to the :func:`connect` -call, or via the :attr:`isolation_level` property of connections. +``autocommit`` mode means that statements that modify the database take effect +immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` +mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the +outermost transaction, turns ``autocommit`` mode back on. -If you want **autocommit mode**, then set :attr:`isolation_level` to ``None``. - -Otherwise leave it at its default, which will result in a plain "BEGIN" -statement, or set it to one of SQLite's supported isolation levels: "DEFERRED", -"IMMEDIATE" or "EXCLUSIVE". +The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement +implicitly before a Data Modification Language (DML) statement (i.e. +``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). -The current transaction state is exposed through the -:attr:`Connection.in_transaction` attribute of the connection object. +You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly +executes via the *isolation_level* parameter to the :func:`connect` +call, or via the :attr:`isolation_level` property of connections. +If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is +equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` +and ``EXCLUSIVE``. + +You can disable the :mod:`sqlite3` module's implicit transaction management by +setting :attr:`isolation_level` to ``None``. This will leave the underlying +``sqlite3`` library operating in ``autocommit`` mode. You can then completely +control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, +``SAVEPOINT``, and ``RELEASE`` statements in your code. .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL diff -Nru python3.7-3.7.0/Doc/library/ssl.rst python3.7-3.7.1/Doc/library/ssl.rst --- python3.7-3.7.0/Doc/library/ssl.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/ssl.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1314,6 +1314,26 @@ returned socket should always be used for further communication with the other side of the connection, rather than the original socket. +.. method:: SSLSocket.verify_client_post_handshake() + + Requests post-handshake authentication (PHA) from a TLS 1.3 client. PHA + can only be initiated for a TLS 1.3 connection from a server-side socket, + after the initial TLS handshake and with PHA enabled on both sides, see + :attr:`SSLContext.post_handshake_auth`. + + The method does not perform a cert exchange immediately. The server-side + sends a CertificateRequest during the next write event and expects the + client to respond with a certificate on the next read event. + + If any precondition isn't met (e.g. not TLS 1.3, PHA not enabled), an + :exc:`SSLError` is raised. + + .. versionadded:: 3.7.1 + + .. note:: + Only available with OpenSSL 1.1.1 and TLS 1.3 enabled. Without TLS 1.3 + support, the method raises :exc:`NotImplementedError`. + .. method:: SSLSocket.version() Return the actual SSL protocol version negotiated by the connection @@ -1607,7 +1627,7 @@ Set the available ciphers for sockets created with this context. It should be a string in the `OpenSSL cipher list format - `_. + `_. If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an :class:`SSLError` will be raised. @@ -1929,6 +1949,28 @@ >>> ssl.create_default_context().options # doctest: +SKIP +.. attribute:: SSLContext.post_handshake_auth + + Enable TLS 1.3 post-handshake client authentication. Post-handshake auth + is disabled by default and a server can only request a TLS client + certificate during the initial handshake. When enabled, a server may + request a TLS client certificate at any time after the handshake. + + When enabled on client-side sockets, the client signals the server that + it supports post-handshake authentication. + + When enabled on server-side sockets, :attr:`SSLContext.verify_mode` must + be set to :data:`CERT_OPTIONAL` or :data:`CERT_REQUIRED`, too. The + actual client cert exchange is delayed until + :meth:`SSLSocket.verify_client_post_handshake` is called and some I/O is + performed. + + .. versionadded:: 3.7.1 + + .. note:: + Only available with OpenSSL 1.1.1 and TLS 1.3 enabled. Without TLS 1.3 + support, the property value is None and can't be modified + .. attribute:: SSLContext.protocol The protocol version chosen when constructing the context. This attribute @@ -1991,7 +2033,7 @@ A certificate contains information about two principals. It contains the name of a *subject*, and the subject's public key. It also contains a statement by a -second principal, the *issuer*, that the subject is who he claims to be, and +second principal, the *issuer*, that the subject is who they claim to be, and that this is indeed the subject's public key. The issuer's statement is signed with the issuer's private key, which only the issuer knows. However, anyone can verify the issuer's statement by finding the issuer's public key, decrypting the @@ -2616,7 +2658,7 @@ - TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and ChaCha20 cipher suites are enabled by default. The method :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3 - ciphers yet, but :meth:`SSLContext.get_cipers` returns them. + ciphers yet, but :meth:`SSLContext.get_ciphers` returns them. - Session tickets are no longer sent as part of the initial handshake and are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession` are not compatible with TLS 1.3. diff -Nru python3.7-3.7.0/Doc/library/statistics.rst python3.7-3.7.1/Doc/library/statistics.rst --- python3.7-3.7.0/Doc/library/statistics.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/statistics.rst 2018-10-20 06:04:19.000000000 +0000 @@ -169,6 +169,10 @@ This is suited for when your data is discrete, and you don't mind that the median may not be an actual data point. + If your data is ordinal (supports order operations) but not numeric (doesn't + support addition), you should use :func:`median_low` or :func:`median_high` + instead. + .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped` diff -Nru python3.7-3.7.0/Doc/library/stdtypes.rst python3.7-3.7.1/Doc/library/stdtypes.rst --- python3.7-3.7.0/Doc/library/stdtypes.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/stdtypes.rst 2018-10-20 06:04:19.000000000 +0000 @@ -197,8 +197,8 @@ operator: not in Two more operations with the same syntactic priority, :keyword:`in` and -:keyword:`not in`, are supported only by sequence types (below). - +:keyword:`not in`, are supported by types that are :term:`iterable` or +implement the :meth:`__contains__` method. .. _typesnumeric: @@ -382,7 +382,7 @@ .. _bitstring-ops: Bitwise Operations on Integer Types --------------------------------------- +----------------------------------- .. index:: triple: operations on; integer; types @@ -396,9 +396,9 @@ operator: >> operator: ~ -Bitwise operations only make sense for integers. Negative numbers are treated -as their 2's complement value (this assumes that there are enough bits so that -no overflow occurs during the operation). +Bitwise operations only make sense for integers. The result of bitwise +operations is calculated as though carried out in two's complement with an +infinite number of sign bits. The priorities of the binary bitwise operations are all lower than the numeric operations and higher than the comparisons; the unary operation ``~`` has the @@ -409,13 +409,13 @@ +------------+--------------------------------+----------+ | Operation | Result | Notes | +============+================================+==========+ -| ``x | y`` | bitwise :dfn:`or` of *x* and | | +| ``x | y`` | bitwise :dfn:`or` of *x* and | \(4) | | | *y* | | +------------+--------------------------------+----------+ -| ``x ^ y`` | bitwise :dfn:`exclusive or` of | | +| ``x ^ y`` | bitwise :dfn:`exclusive or` of | \(4) | | | *x* and *y* | | +------------+--------------------------------+----------+ -| ``x & y`` | bitwise :dfn:`and` of *x* and | | +| ``x & y`` | bitwise :dfn:`and` of *x* and | \(4) | | | *y* | | +------------+--------------------------------+----------+ | ``x << n`` | *x* shifted left by *n* bits | (1)(2) | @@ -438,6 +438,12 @@ A right shift by *n* bits is equivalent to division by ``pow(2, n)`` without overflow check. +(4) + Performing these calculations with at least one extra sign extension bit in + a finite two's complement representation (a working bit-width of + ``1 + max(x.bit_length(), y.bit_length()`` or more) is sufficient to get the + same result as if there were an infinite number of sign bits. + Additional Methods on Integer Types ----------------------------------- @@ -1059,10 +1065,10 @@ | | sequence (same as | | | | ``s[len(s):len(s)] = [x]``) | | +------------------------------+--------------------------------+---------------------+ -| ``s.clear()`` | removes all items from ``s`` | \(5) | +| ``s.clear()`` | removes all items from *s* | \(5) | | | (same as ``del s[:]``) | | +------------------------------+--------------------------------+---------------------+ -| ``s.copy()`` | creates a shallow copy of ``s``| \(5) | +| ``s.copy()`` | creates a shallow copy of *s* | \(5) | | | (same as ``s[:]``) | | +------------------------------+--------------------------------+---------------------+ | ``s.extend(t)`` or | extends *s* with the | | @@ -1365,7 +1371,7 @@ .. seealso:: * The `linspace recipe `_ - shows how to implement a lazy version of range that suitable for floating + shows how to implement a lazy version of range suitable for floating point applications. .. index:: @@ -1600,13 +1606,14 @@ that can be specified in format strings. .. note:: - When formatting a number (:class:`int`, :class:`float`, :class:`float` - and subclasses) with the ``n`` type (ex: ``'{:n}'.format(1234)``), the - function sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` - locale to decode ``decimal_point`` and ``thousands_sep`` fields of - :c:func:`localeconv` if they are non-ASCII or longer than 1 byte, and the - ``LC_NUMERIC`` locale is different than the ``LC_CTYPE`` locale. This - temporary change affects other threads. + When formatting a number (:class:`int`, :class:`float`, :class:`complex`, + :class:`decimal.Decimal` and subclasses) with the ``n`` type + (ex: ``'{:n}'.format(1234)``), the function temporarily sets the + ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale to decode + ``decimal_point`` and ``thousands_sep`` fields of :c:func:`localeconv` if + they are non-ASCII or longer than 1 byte, and the ``LC_NUMERIC`` locale is + different than the ``LC_CTYPE`` locale. This temporary change affects + other threads. .. versionchanged:: 3.7 When formatting a number with the ``n`` type, the function sets @@ -2051,7 +2058,7 @@ .. method:: str.upper() Return a copy of the string with all the cased characters [4]_ converted to - uppercase. Note that ``str.upper().isupper()`` might be ``False`` if ``s`` + uppercase. Note that ``s.upper().isupper()`` might be ``False`` if ``s`` contains uncased characters or if the Unicode category of the resulting character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter, titlecase). @@ -2315,7 +2322,7 @@ While bytes literals and representations are based on ASCII text, bytes objects actually behave like immutable sequences of integers, with each value in the sequence restricted such that ``0 <= x < 256`` (attempts to - violate this restriction will trigger :exc:`ValueError`. This is done + violate this restriction will trigger :exc:`ValueError`). This is done deliberately to emphasise that while many binary formats include ASCII based elements and can be usefully manipulated with some text-oriented algorithms, this is not generally the case for arbitrary binary data (blindly applying @@ -3388,7 +3395,10 @@ The bytearray version of this method does *not* operate in place - it always produces a new object, even if no changes were made. -.. seealso:: :pep:`461`. +.. seealso:: + + :pep:`461` - Adding % formatting to bytes and bytearray + .. versionadded:: 3.5 .. _typememoryview: @@ -4198,12 +4208,17 @@ .. method:: popitem() - Remove and return an arbitrary ``(key, value)`` pair from the dictionary. + Remove and return a ``(key, value)`` pair from the dictionary. + Pairs are returned in :abbr:`LIFO (last-in, first-out)` order. :meth:`popitem` is useful to destructively iterate over a dictionary, as often used in set algorithms. If the dictionary is empty, calling :meth:`popitem` raises a :exc:`KeyError`. + .. versionchanged:: 3.7 + LIFO order is now guaranteed. In prior versions, :meth:`popitem` would + return an arbitrary key/value pair. + .. method:: setdefault(key[, default]) If *key* is in the dictionary, return its value. If not, insert *key* diff -Nru python3.7-3.7.0/Doc/library/string.rst python3.7-3.7.1/Doc/library/string.rst --- python3.7-3.7.0/Doc/library/string.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/string.rst 2018-10-20 06:04:19.000000000 +0000 @@ -231,8 +231,11 @@ does an index lookup using :func:`__getitem__`. .. versionchanged:: 3.1 - The positional argument specifiers can be omitted, so ``'{} {}'`` is - equivalent to ``'{0} {1}'``. + The positional argument specifiers can be omitted for :meth:`str.format`, + so ``'{} {}'.format(a, b)`` is equivalent to ``'{0} {1}'.format(a, b)``. + +.. versionchanged:: 3.4 + The positional argument specifiers can be omitted for :class:`Formatter`. Some simple format string examples:: @@ -461,11 +464,11 @@ | ``'E'`` | Exponent notation. Same as ``'e'`` except it uses an | | | upper case 'E' as the separator character. | +---------+----------------------------------------------------------+ - | ``'f'`` | Fixed point. Displays the number as a fixed-point | - | | number. The default precision is ``6``. | + | ``'f'`` | Fixed-point notation. Displays the number as a | + | | fixed-point number. The default precision is ``6``. | +---------+----------------------------------------------------------+ - | ``'F'`` | Fixed point. Same as ``'f'``, but converts ``nan`` to | - | | ``NAN`` and ``inf`` to ``INF``. | + | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | + | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | +---------+----------------------------------------------------------+ | ``'g'`` | General format. For a given precision ``p >= 1``, | | | this rounds the number to ``p`` significant digits and | diff -Nru python3.7-3.7.0/Doc/library/struct.rst python3.7-3.7.1/Doc/library/struct.rst --- python3.7-3.7.0/Doc/library/struct.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/struct.rst 2018-10-20 06:04:19.000000000 +0000 @@ -405,6 +405,12 @@ methods is more efficient than calling the :mod:`struct` functions with the same format since the format string only needs to be compiled once. + .. note:: + + The compiled versions of the most recent format strings passed to + :class:`Struct` and the module-level functions are cached, so programs + that use only a few format strings needn't worry about reusing a single + :class:`Struct` instance. Compiled Struct objects support the following methods and attributes: diff -Nru python3.7-3.7.0/Doc/library/subprocess.rst python3.7-3.7.1/Doc/library/subprocess.rst --- python3.7-3.7.0/Doc/library/subprocess.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/subprocess.rst 2018-10-20 06:04:19.000000000 +0000 @@ -38,8 +38,8 @@ .. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ - shell=False, cwd=None, timeout=None, check=False, \ - encoding=None, errors=None, text=None, env=None) + capture_output=False, shell=False, cwd=None, timeout=None, \ + check=False, encoding=None, errors=None, text=None, env=None) Run the command described by *args*. Wait for command to complete, then return a :class:`CompletedProcess` instance. @@ -1336,14 +1336,15 @@ Windows support was added. The function now returns (exitcode, output) instead of (status, output) - as it did in Python 3.3.3 and earlier. See :func:`WEXITSTATUS`. + as it did in Python 3.3.3 and earlier. exitcode has the same value as + :attr:`~Popen.returncode`. .. function:: getoutput(cmd) Return output (stdout and stderr) of executing *cmd* in a shell. - Like :func:`getstatusoutput`, except the exit status is ignored and the return + Like :func:`getstatusoutput`, except the exit code is ignored and the return value is a string containing the command's output. Example:: >>> subprocess.getoutput('ls /bin/ls') diff -Nru python3.7-3.7.0/Doc/library/syslog.rst python3.7-3.7.1/Doc/library/syslog.rst --- python3.7-3.7.0/Doc/library/syslog.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/syslog.rst 2018-10-20 06:04:19.000000000 +0000 @@ -48,7 +48,7 @@ .. versionchanged:: 3.2 In previous versions, keyword arguments were not allowed, and *ident* was required. The default for *ident* was dependent on the system libraries, - and often was ``python`` instead of the name of the python program file. + and often was ``python`` instead of the name of the Python program file. .. function:: closelog() diff -Nru python3.7-3.7.0/Doc/library/sys.rst python3.7-3.7.1/Doc/library/sys.rst --- python3.7-3.7.0/Doc/library/sys.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/sys.rst 2018-10-20 06:04:19.000000000 +0000 @@ -313,9 +313,6 @@ has caught :exc:`SystemExit` (such as an error flushing buffered data in the standard streams), the exit status is changed to 120. - .. versionchanged:: 3.7 - Added ``utf8_mode`` attribute for the new :option:`-X` ``utf8`` flag. - .. data:: flags @@ -328,6 +325,7 @@ :const:`debug` :option:`-d` :const:`inspect` :option:`-i` :const:`interactive` :option:`-i` + :const:`isolated` :option:`-I` :const:`optimize` :option:`-O` or :option:`-OO` :const:`dont_write_bytecode` :option:`-B` :const:`no_user_site` :option:`-s` @@ -350,6 +348,9 @@ .. versionchanged:: 3.3 Removed obsolete ``division_warning`` attribute. + .. versionchanged:: 3.4 + Added ``isolated`` attribute for :option:`-I` ``isolated`` flag. + .. versionchanged:: 3.7 Added ``dev_mode`` attribute for the new :option:`-X` ``dev`` flag and ``utf8_mode`` attribute for the new :option:`-X` ``utf8`` flag. diff -Nru python3.7-3.7.0/Doc/library/tempfile.rst python3.7-3.7.1/Doc/library/tempfile.rst --- python3.7-3.7.0/Doc/library/tempfile.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/tempfile.rst 2018-10-20 06:04:19.000000000 +0000 @@ -253,7 +253,7 @@ default value for the *dir* argument to the functions defined in this module. - If ``tempdir`` is unset or ``None`` at any call to any of the above + If ``tempdir`` is ``None`` (the default) at any call to any of the above functions except :func:`gettempprefix` it is initialized following the algorithm described in :func:`gettempdir`. diff -Nru python3.7-3.7.0/Doc/library/test.rst python3.7-3.7.1/Doc/library/test.rst --- python3.7-3.7.0/Doc/library/test.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/test.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1076,7 +1076,7 @@ Either this method or :func:`bind_port` should be used for any tests where a server socket needs to be bound to a particular port for the duration of the test. - Which one to use depends on whether the calling code is creating a python + Which one to use depends on whether the calling code is creating a Python socket, or if an unused port needs to be provided in a constructor or passed to an external program (i.e. the ``-accept`` argument to openssl's s_server mode). Always prefer :func:`bind_port` over diff -Nru python3.7-3.7.0/Doc/library/threading.rst python3.7-3.7.1/Doc/library/threading.rst --- python3.7-3.7.0/Doc/library/threading.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/threading.rst 2018-10-20 06:04:19.000000000 +0000 @@ -400,7 +400,8 @@ The *timeout* parameter is new. .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + Lock acquisition can now be interrupted by signals on POSIX if the + underlying threading implementation supports it. .. method:: release() diff -Nru python3.7-3.7.0/Doc/library/traceback.rst python3.7-3.7.1/Doc/library/traceback.rst --- python3.7-3.7.0/Doc/library/traceback.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/traceback.rst 2018-10-20 06:04:19.000000000 +0000 @@ -88,14 +88,16 @@ .. function:: extract_tb(tb, limit=None) - Return a list of "pre-processed" stack trace entries extracted from the - traceback object *tb*. It is useful for alternate formatting of - stack traces. The optional *limit* argument has the same meaning as for - :func:`print_tb`. A "pre-processed" stack trace entry is a 4-tuple - (*filename*, *line number*, *function name*, *text*) representing the - information that is usually printed for a stack trace. The *text* is a - string with leading and trailing whitespace stripped; if the source is - not available it is ``None``. + Return a :class:`StackSummary` object representing a list of "pre-processed" + stack trace entries extracted from the traceback object *tb*. It is useful + for alternate formatting of stack traces. The optional *limit* argument has + the same meaning as for :func:`print_tb`. A "pre-processed" stack trace + entry is a :class:`FrameSummary` object containing attributes + :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, + :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the + information that is usually printed for a stack trace. The + :attr:`~FrameSummary.line` is a string with leading and trailing + whitespace stripped; if the source is not available it is ``None``. .. function:: extract_stack(f=None, limit=None) @@ -107,12 +109,12 @@ .. function:: format_list(extracted_list) - Given a list of tuples as returned by :func:`extract_tb` or - :func:`extract_stack`, return a list of strings ready for printing. Each - string in the resulting list corresponds to the item with the same index in - the argument list. Each string ends in a newline; the strings may contain - internal newlines as well, for those items whose source text line is not - ``None``. + Given a list of tuples or :class:`FrameSummary` objects as returned by + :func:`extract_tb` or :func:`extract_stack`, return a list of strings ready + for printing. Each string in the resulting list corresponds to the item with + the same index in the argument list. Each string ends in a newline; the + strings may contain internal newlines as well, for those items whose source + text line is not ``None``. .. function:: format_exception_only(etype, value) @@ -293,9 +295,9 @@ .. classmethod:: from_list(a_list) - Construct a :class:`StackSummary` object from a supplied old-style list - of tuples. Each tuple should be a 4-tuple with filename, lineno, name, - line as the elements. + Construct a :class:`StackSummary` object from a supplied list of + :class:`FrameSummary` objects or old-style list of tuples. Each tuple + should be a 4-tuple with filename, lineno, name, line as the elements. .. method:: format() diff -Nru python3.7-3.7.0/Doc/library/typing.rst python3.7-3.7.1/Doc/library/typing.rst --- python3.7-3.7.0/Doc/library/typing.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/typing.rst 2018-10-20 06:04:19.000000000 +0000 @@ -169,6 +169,8 @@ the call signature by substituting a literal ellipsis for the list of arguments in the type hint: ``Callable[..., ReturnType]``. +.. _generics: + Generics -------- @@ -183,7 +185,7 @@ def notify_by_email(employees: Sequence[Employee], overrides: Mapping[str, str]) -> None: ... -Generics can be parametrized by using a new factory available in typing +Generics can be parameterized by using a new factory available in typing called :class:`TypeVar`. :: @@ -488,8 +490,9 @@ required to handle this particular case may change in future revisions of :pep:`484`. - The only legal parameters for :class:`Type` are classes, unions of classes, and - :data:`Any`. For example:: + The only legal parameters for :class:`Type` are classes, :data:`Any`, + :ref:`type variables `, and unions of any of these types. + For example:: def new_non_team_user(user_class: Type[Union[BaseUser, ProUser]]): ... @@ -990,10 +993,18 @@ Note that this is not the same concept as an optional argument, which is one that has a default. An optional argument with a - default needn't use the ``Optional`` qualifier on its type - annotation (although it is inferred if the default is ``None``). - A mandatory argument may still have an ``Optional`` type if an - explicit value of ``None`` is allowed. + default does not require the ``Optional`` qualifier on its type + annotation just because it is optional. For example:: + + def foo(arg: int = 0) -> None: + ... + + On the other hand, if an explicit value of ``None`` is allowed, the + use of ``Optional`` is appropriate, whether the argument is optional + or not. For example:: + + def foo(arg: Optional[int] = None) -> None: + ... .. data:: Tuple diff -Nru python3.7-3.7.0/Doc/library/unittest.mock-examples.rst python3.7-3.7.1/Doc/library/unittest.mock-examples.rst --- python3.7-3.7.0/Doc/library/unittest.mock-examples.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/unittest.mock-examples.rst 2018-10-20 06:04:19.000000000 +0000 @@ -392,7 +392,7 @@ >>> MyTest('test_something').test_something() When you nest patch decorators the mocks are passed in to the decorated -function in the same order they applied (the normal *python* order that +function in the same order they applied (the normal *Python* order that decorators are applied). This means from the bottom up, so in the example above the mock for ``test_module.ClassName2`` is passed in first. diff -Nru python3.7-3.7.0/Doc/library/unittest.mock.rst python3.7-3.7.1/Doc/library/unittest.mock.rst --- python3.7-3.7.0/Doc/library/unittest.mock.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/unittest.mock.rst 2018-10-20 06:04:19.000000000 +0000 @@ -98,7 +98,7 @@ .. note:: When you nest patch decorators the mocks are passed in to the decorated - function in the same order they applied (the normal *python* order that + function in the same order they applied (the normal *Python* order that decorators are applied). This means from the bottom up, so in the example above the mock for ``module.ClassName1`` is passed in first. @@ -1825,12 +1825,12 @@ .. data:: sentinel - The ``sentinel`` object provides a convenient way of providing unique - objects for your tests. + The ``sentinel`` object provides a convenient way of providing unique + objects for your tests. - Attributes are created on demand when you access them by name. Accessing - the same attribute will always return the same object. The objects - returned have a sensible repr so that test failure messages are readable. + Attributes are created on demand when you access them by name. Accessing + the same attribute will always return the same object. The objects + returned have a sensible repr so that test failure messages are readable. .. versionchanged:: 3.7 The ``sentinel`` attributes now preserve their identity when they are @@ -2070,22 +2070,22 @@ .. function:: mock_open(mock=None, read_data=None) - A helper function to create a mock to replace the use of :func:`open`. It works - for :func:`open` called directly or used as a context manager. + A helper function to create a mock to replace the use of :func:`open`. It works + for :func:`open` called directly or used as a context manager. - The *mock* argument is the mock object to configure. If ``None`` (the - default) then a :class:`MagicMock` will be created for you, with the API limited - to methods or attributes available on standard file handles. - - *read_data* is a string for the :meth:`~io.IOBase.read`, - :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods - of the file handle to return. Calls to those methods will take data from - *read_data* until it is depleted. The mock of these methods is pretty - simplistic: every time the *mock* is called, the *read_data* is rewound to - the start. If you need more control over the data that you are feeding to - the tested code you will need to customize this mock for yourself. When that - is insufficient, one of the in-memory filesystem packages on `PyPI - `_ can offer a realistic filesystem for testing. + The *mock* argument is the mock object to configure. If ``None`` (the + default) then a :class:`MagicMock` will be created for you, with the API limited + to methods or attributes available on standard file handles. + + *read_data* is a string for the :meth:`~io.IOBase.read`, + :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods + of the file handle to return. Calls to those methods will take data from + *read_data* until it is depleted. The mock of these methods is pretty + simplistic: every time the *mock* is called, the *read_data* is rewound to + the start. If you need more control over the data that you are feeding to + the tested code you will need to customize this mock for yourself. When that + is insufficient, one of the in-memory filesystem packages on `PyPI + `_ can offer a realistic filesystem for testing. .. versionchanged:: 3.4 Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support. @@ -2095,6 +2095,10 @@ .. versionchanged:: 3.5 *read_data* is now reset on each call to the *mock*. + .. versionchanged:: 3.7.1 + Added :meth:`__iter__` to implementation so that iteration (such as in for + loops) correctly consumes *read_data*. + Using :func:`open` as a context manager is a great way to ensure your file handles are closed properly and is becoming common:: diff -Nru python3.7-3.7.0/Doc/library/unittest.rst python3.7-3.7.1/Doc/library/unittest.rst --- python3.7-3.7.0/Doc/library/unittest.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/unittest.rst 2018-10-20 06:04:19.000000000 +0000 @@ -412,7 +412,7 @@ Such a working environment for the testing code is called a :dfn:`test fixture`. A new TestCase instance is created as a unique test fixture used to execute each individual test method. Thus -`~TestCase.setUp`, `~TestCase.tearDown`, and `~TestCase.__init__` +:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`~TestCase.__init__` will be called once per test. It is recommended that you use TestCase implementations to group tests together @@ -724,7 +724,7 @@ .. method:: setUpClass() - A class method called before tests in an individual class run. + A class method called before tests in an individual class are run. ``setUpClass`` is called with the class as the only argument and must be decorated as a :func:`classmethod`:: @@ -944,7 +944,7 @@ +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) - assertRaises(exception, msg=None) + assertRaises(exception, *, msg=None) Test that an exception is raised when *callable* is called with any positional or keyword arguments that are also passed to @@ -984,7 +984,7 @@ .. method:: assertRaisesRegex(exception, regex, callable, *args, **kwds) - assertRaisesRegex(exception, regex, msg=None) + assertRaisesRegex(exception, regex, *, msg=None) Like :meth:`assertRaises` but also tests that *regex* matches on the string representation of the raised exception. *regex* may be @@ -1010,7 +1010,7 @@ .. method:: assertWarns(warning, callable, *args, **kwds) - assertWarns(warning, msg=None) + assertWarns(warning, *, msg=None) Test that a warning is triggered when *callable* is called with any positional or keyword arguments that are also passed to @@ -1051,7 +1051,7 @@ .. method:: assertWarnsRegex(warning, regex, callable, *args, **kwds) - assertWarnsRegex(warning, regex, msg=None) + assertWarnsRegex(warning, regex, *, msg=None) Like :meth:`assertWarns` but also tests that *regex* matches on the message of the triggered warning. *regex* may be a regular expression diff -Nru python3.7-3.7.0/Doc/library/urllib.request.rst python3.7-3.7.1/Doc/library/urllib.request.rst --- python3.7-3.7.0/Doc/library/urllib.request.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/urllib.request.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1125,7 +1125,7 @@ HTTPErrorProcessor Objects -------------------------- -.. method:: HTTPErrorProcessor.http_response() +.. method:: HTTPErrorProcessor.http_response(request, response) Process HTTP error responses. @@ -1137,7 +1137,7 @@ :exc:`~urllib.error.HTTPError` if no other handler handles the error. -.. method:: HTTPErrorProcessor.https_response() +.. method:: HTTPErrorProcessor.https_response(request, response) Process HTTPS error responses. @@ -1339,9 +1339,9 @@ The second argument, if present, specifies the file location to copy to (if absent, the location will be a tempfile with a generated name). The third - argument, if present, is a hook function that will be called once on + argument, if present, is a callable that will be called once on establishment of the network connection and once after each block read - thereafter. The hook will be passed three arguments; a count of blocks + thereafter. The callable will be passed three arguments; a count of blocks transferred so far, a block size in bytes, and the total size of the file. The third argument may be ``-1`` on older FTP servers which do not return a file size in response to a retrieval request. diff -Nru python3.7-3.7.0/Doc/library/venv.rst python3.7-3.7.1/Doc/library/venv.rst --- python3.7-3.7.0/Doc/library/venv.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/venv.rst 2018-10-20 06:04:19.000000000 +0000 @@ -17,12 +17,18 @@ The :mod:`venv` module provides support for creating lightweight "virtual environments" with their own site directories, optionally isolated from system -site directories. Each virtual environment has its own Python binary (allowing -creation of environments with various Python versions) and can have its own -independent set of installed Python packages in its site directories. +site directories. Each virtual environment has its own Python binary (which +matches the version of the binary that was used to create this environment) and +can have its own independent set of installed Python packages in its site +directories. See :pep:`405` for more information about Python virtual environments. +.. seealso:: + + `Python Packaging User Guide: Creating and using virtual environments + `__ + .. note:: The ``pyvenv`` script has been deprecated as of Python 3.6 in favor of using ``python3 -m venv`` to help prevent any potential confusion as to which diff -Nru python3.7-3.7.0/Doc/library/xml.dom.pulldom.rst python3.7-3.7.1/Doc/library/xml.dom.pulldom.rst --- python3.7-3.7.0/Doc/library/xml.dom.pulldom.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/xml.dom.pulldom.rst 2018-10-20 06:04:19.000000000 +0000 @@ -25,6 +25,20 @@ maliciously constructed data. If you need to parse untrusted or unauthenticated data see :ref:`xml-vulnerabilities`. +.. versionchanged:: 3.7.1 + + The SAX parser no longer processes general external entities by default to + increase security by default. To enable processing of external entities, + pass a custom parser instance in:: + + from xml.dom.pulldom import parse + from xml.sax import make_parser + from xml.sax.handler import feature_external_ges + + parser = make_parser() + parser.setFeature(feature_external_ges, True) + parse(filename, parser=parser) + Example:: diff -Nru python3.7-3.7.0/Doc/library/xmlrpc.client.rst python3.7-3.7.1/Doc/library/xmlrpc.client.rst --- python3.7-3.7.0/Doc/library/xmlrpc.client.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/xmlrpc.client.rst 2018-10-20 06:04:19.000000000 +0000 @@ -145,7 +145,7 @@ .. versionchanged:: 3.6 Added support of type tags with prefixes (e.g. ``ex:nil``). - Added support of unmarsalling additional types used by Apache XML-RPC + Added support of unmarshalling additional types used by Apache XML-RPC implementation for numerics: ``i1``, ``i2``, ``i8``, ``biginteger``, ``float`` and ``bigdecimal``. See http://ws.apache.org/xmlrpc/types.html for a description. diff -Nru python3.7-3.7.0/Doc/library/xml.rst python3.7-3.7.1/Doc/library/xml.rst --- python3.7-3.7.0/Doc/library/xml.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/xml.rst 2018-10-20 06:04:19.000000000 +0000 @@ -65,8 +65,8 @@ ========================= ============== =============== ============== ============== ============== billion laughs **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** quadratic blowup **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -external entity expansion **Vulnerable** Safe (1) Safe (2) **Vulnerable** Safe (3) -`DTD`_ retrieval **Vulnerable** Safe Safe **Vulnerable** Safe +external entity expansion Safe (4) Safe (1) Safe (2) Safe (4) Safe (3) +`DTD`_ retrieval Safe (4) Safe Safe Safe (4) Safe decompression bomb Safe Safe Safe Safe **Vulnerable** ========================= ============== =============== ============== ============== ============== @@ -75,6 +75,8 @@ 2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns the unexpanded entity verbatim. 3. :mod:`xmlrpclib` doesn't expand external entities and omits them. +4. Since Python 3.8.0, external general entities are no longer processed by + default since Python. billion laughs / exponential entity expansion diff -Nru python3.7-3.7.0/Doc/library/xml.sax.rst python3.7-3.7.1/Doc/library/xml.sax.rst --- python3.7-3.7.0/Doc/library/xml.sax.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/xml.sax.rst 2018-10-20 06:04:19.000000000 +0000 @@ -24,6 +24,14 @@ constructed data. If you need to parse untrusted or unauthenticated data see :ref:`xml-vulnerabilities`. +.. versionchanged:: 3.7.1 + + The SAX parser no longer processes general external entities by default + to increase security. Before, the parser created network connections + to fetch remote files or loaded local files from the file + system for DTD and entities. The feature can be enabled again with method + :meth:`~xml.sax.xmlreader.XMLReader.setFeature` on the parser object + and argument :data:`~xml.sax.handler.feature_external_ges`. The convenience functions are: diff -Nru python3.7-3.7.0/Doc/library/zipapp.rst python3.7-3.7.1/Doc/library/zipapp.rst --- python3.7-3.7.0/Doc/library/zipapp.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/library/zipapp.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1,8 +1,8 @@ -:mod:`zipapp` --- Manage executable python zip archives +:mod:`zipapp` --- Manage executable Python zip archives ======================================================= .. module:: zipapp - :synopsis: Manage executable python zip archives + :synopsis: Manage executable Python zip archives .. versionadded:: 3.5 @@ -195,7 +195,7 @@ $ python myapp.pyz -The same can be done using the :func:`create_archive` functon:: +The same can be done using the :func:`create_archive` function:: >>> import zipapp >>> zipapp.create_archive('myapp.pyz', 'myapp') diff -Nru python3.7-3.7.0/Doc/make.bat python3.7-3.7.1/Doc/make.bat --- python3.7-3.7.0/Doc/make.bat 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/make.bat 2018-10-20 06:04:19.000000000 +0000 @@ -131,6 +131,9 @@ if NOT "%PAPER%" == "" ( set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% ) +if "%1" EQU "htmlhelp" ( + set SPHINXOPTS=-D html_theme_options.body_max_width=none %SPHINXOPTS% +) cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . "%BUILDDIR%\%1" %2 %3 %4 %5 %6 %7 %8 %9" if "%1" EQU "htmlhelp" ( diff -Nru python3.7-3.7.0/Doc/README.rst python3.7-3.7.1/Doc/README.rst --- python3.7-3.7.0/Doc/README.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/README.rst 2018-10-20 06:04:19.000000000 +0000 @@ -33,7 +33,7 @@ make venv That will install all the tools necessary to build the documentation. Assuming -the virtual environment was created in the ``env`` directory (the default; +the virtual environment was created in the ``venv`` directory (the default; configurable with the VENVDIR variable), you can run the following command to build the HTML output files:: diff -Nru python3.7-3.7.0/Doc/reference/compound_stmts.rst python3.7-3.7.1/Doc/reference/compound_stmts.rst --- python3.7-3.7.0/Doc/reference/compound_stmts.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/reference/compound_stmts.rst 2018-10-20 06:04:19.000000000 +0000 @@ -91,7 +91,7 @@ .. productionlist:: if_stmt: "if" `expression` ":" `suite` - : ( "elif" `expression` ":" `suite` )* + : ("elif" `expression` ":" `suite`)* : ["else" ":" `suite`] It selects exactly one of the suites by evaluating the expressions one by one @@ -203,7 +203,7 @@ single: mutable sequence; loop over There is a subtlety when the sequence is being modified by the loop (this can - only occur for mutable sequences, i.e. lists). An internal counter is used + only occur for mutable sequences, e.g. lists). An internal counter is used to keep track of which item is used next, and this is incremented on each iteration. When this counter has reached the length of the sequence the loop terminates. This means that if the suite deletes the current (or a previous) @@ -235,7 +235,7 @@ for a group of statements: .. productionlist:: - try_stmt: try1_stmt | try2_stmt + try_stmt: `try1_stmt` | `try2_stmt` try1_stmt: "try" ":" `suite` : ("except" [`expression` ["as" `identifier`]] ":" `suite`)+ : ["else" ":" `suite`] @@ -383,7 +383,7 @@ usage patterns to be encapsulated for convenient reuse. .. productionlist:: - with_stmt: "with" with_item ("," with_item)* ":" `suite` + with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite` with_item: `expression` ["as" `target`] The execution of the :keyword:`with` statement with one "item" proceeds as follows: @@ -467,14 +467,15 @@ :ref:`types`): .. productionlist:: - funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" ["->" `expression`] ":" `suite` + funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" + : ["->" `expression`] ":" `suite` decorators: `decorator`+ decorator: "@" `dotted_name` ["(" [`argument_list` [","]] ")"] NEWLINE dotted_name: `identifier` ("." `identifier`)* parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] : | `parameter_list_starargs` parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] - : | "**" `parameter` [","] + : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -697,7 +698,8 @@ ----------------------------- .. productionlist:: - async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" ["->" `expression`] ":" `suite` + async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" + : ["->" `expression`] ":" `suite` .. index:: keyword: async diff -Nru python3.7-3.7.0/Doc/reference/datamodel.rst python3.7-3.7.1/Doc/reference/datamodel.rst --- python3.7-3.7.0/Doc/reference/datamodel.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/reference/datamodel.rst 2018-10-20 06:04:19.000000000 +0000 @@ -1059,9 +1059,9 @@ trace (towards the frame where the exception occurred), or ``None`` if there is no next level. - .. versionchanged:: 3.7 - Traceback objects can now be explicitly instantiated from Python code, - and the ``tb_next`` attribute of existing instances can be updated. + .. versionchanged:: 3.7 + Traceback objects can now be explicitly instantiated from Python code, + and the ``tb_next`` attribute of existing instances can be updated. Slice objects .. index:: builtin: slice diff -Nru python3.7-3.7.0/Doc/reference/expressions.rst python3.7-3.7.1/Doc/reference/expressions.rst --- python3.7-3.7.0/Doc/reference/expressions.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/reference/expressions.rst 2018-10-20 06:04:19.000000000 +0000 @@ -777,7 +777,7 @@ value in the mapping that corresponds to that key. (The expression list is a tuple except if it has exactly one item.) -If the primary is a sequence, the expression (list) must evaluate to an integer +If the primary is a sequence, the expression list must evaluate to an integer or a slice (as discussed in the following section). The formal syntax makes no special provision for negative indices in @@ -1060,7 +1060,7 @@ less tightly than unary operators on its right. The syntax is: .. productionlist:: - power: ( `await_expr` | `primary` ) ["**" `u_expr`] + power: (`await_expr` | `primary`) ["**" `u_expr`] Thus, in an unparenthesized sequence of power and unary operators, the operators are evaluated from right to left (this does not constrain the evaluation order @@ -1132,7 +1132,7 @@ .. productionlist:: m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` | - : `m_expr` "//" `u_expr`| `m_expr` "/" `u_expr` | + : `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` | : `m_expr` "%" `u_expr` a_expr: `m_expr` | `a_expr` "+" `m_expr` | `a_expr` "-" `m_expr` @@ -1144,7 +1144,9 @@ common type and then multiplied together. In the latter case, sequence repetition is performed; a negative repetition factor yields an empty sequence. -.. index:: single: matrix multiplication +.. index:: + single: matrix multiplication + operator: @ The ``@`` (at) operator is intended to be used for matrix multiplication. No builtin Python types implement this operator. @@ -1210,7 +1212,7 @@ The shifting operations have lower priority than the arithmetic operations: .. productionlist:: - shift_expr: `a_expr` | `shift_expr` ( "<<" | ">>" ) `a_expr` + shift_expr: `a_expr` | `shift_expr` ("<<" | ">>") `a_expr` These operators accept integers as arguments. They shift the first argument to the left or right by the number of bits given by the second argument. @@ -1270,7 +1272,7 @@ in mathematics: .. productionlist:: - comparison: `or_expr` ( `comp_operator` `or_expr` )* + comparison: `or_expr` (`comp_operator` `or_expr`)* comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" : | "is" ["not"] | ["not"] "in" @@ -1339,12 +1341,11 @@ involved, they compare mathematically (algorithmically) correct without loss of precision. - The not-a-number values :const:`float('NaN')` and :const:`Decimal('NaN')` - are special. They are identical to themselves (``x is x`` is true) but - are not equal to themselves (``x == x`` is false). Additionally, - comparing any number to a not-a-number value - will return ``False``. For example, both ``3 < float('NaN')`` and - ``float('NaN') < 3`` will return ``False``. + The not-a-number values ``float('NaN')`` and ``decimal.Decimal('NaN')`` are + special. Any ordered comparison of a number to a not-a-number value is false. + A counter-intuitive implication is that not-a-number values are not equal to + themselves. For example, if ``x = float('NaN')``, ``3 < x``, ``x < 3``, ``x + == x``, ``x != x`` are all false. This behavior is compliant with IEEE 754. * Binary sequences (instances of :class:`bytes` or :class:`bytearray`) can be compared within and across their types. They compare lexicographically using @@ -1634,9 +1635,9 @@ .. index:: pair: expression; list .. productionlist:: - expression_list: `expression` ( "," `expression` )* [","] - starred_list: `starred_item` ( "," `starred_item` )* [","] - starred_expression: `expression` | ( `starred_item` "," )* [`starred_item`] + expression_list: `expression` ("," `expression`)* [","] + starred_list: `starred_item` ("," `starred_item`)* [","] + starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] starred_item: `expression` | "*" `or_expr` .. index:: object: tuple diff -Nru python3.7-3.7.0/Doc/reference/simple_stmts.rst python3.7-3.7.1/Doc/reference/simple_stmts.rst --- python3.7-3.7.0/Doc/reference/simple_stmts.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/reference/simple_stmts.rst 2018-10-20 06:04:19.000000000 +0000 @@ -708,15 +708,14 @@ keyword: from .. productionlist:: - import_stmt: "import" `module` ["as" `name`] ( "," `module` ["as" `name`] )* - : | "from" `relative_module` "import" `identifier` ["as" `name`] - : ( "," `identifier` ["as" `name`] )* - : | "from" `relative_module` "import" "(" `identifier` ["as" `name`] - : ( "," `identifier` ["as" `name`] )* [","] ")" + import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* + : | "from" `relative_module` "import" `identifier` ["as" `identifier`] + : ("," `identifier` ["as" `identifier`])* + : | "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] + : ("," `identifier` ["as" `identifier`])* [","] ")" : | "from" `module` "import" "*" module: (`identifier` ".")* `identifier` relative_module: "."* `module` | "."+ - name: `identifier` The basic import statement (no :keyword:`from` clause) is executed in two steps: @@ -838,12 +837,11 @@ standard. .. productionlist:: * - future_statement: "from" "__future__" "import" feature ["as" name] - : ("," feature ["as" name])* - : | "from" "__future__" "import" "(" feature ["as" name] - : ("," feature ["as" name])* [","] ")" - feature: identifier - name: identifier + future_stmt: "from" "__future__" "import" `feature` ["as" `identifier`] + : ("," `feature` ["as" `identifier`])* + : | "from" "__future__" "import" "(" `feature` ["as" `identifier`] + : ("," `feature` ["as" `identifier`])* [","] ")" + feature: `identifier` A future statement must appear near the top of the module. The only lines that can appear before a future statement are: diff -Nru python3.7-3.7.0/Doc/reference/toplevel_components.rst python3.7-3.7.1/Doc/reference/toplevel_components.rst --- python3.7-3.7.0/Doc/reference/toplevel_components.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/reference/toplevel_components.rst 2018-10-20 06:04:19.000000000 +0000 @@ -48,14 +48,15 @@ .. index:: single: UNIX + single: Windows single: command line single: standard input -Under Unix, a complete program can be passed to the interpreter in three forms: -with the :option:`-c` *string* command line option, as a file passed as the -first command line argument, or as standard input. If the file or standard -input is a tty device, the interpreter enters interactive mode; otherwise, it -executes the file as a complete program. +A complete program can be passed to the interpreter +in three forms: with the :option:`-c` *string* command line option, as a file +passed as the first command line argument, or as standard input. If the file +or standard input is a tty device, the interpreter enters interactive mode; +otherwise, it executes the file as a complete program. .. _file-input: diff -Nru python3.7-3.7.0/Doc/tools/extensions/escape4chm.py python3.7-3.7.1/Doc/tools/extensions/escape4chm.py --- python3.7-3.7.0/Doc/tools/extensions/escape4chm.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Doc/tools/extensions/escape4chm.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,39 @@ +""" +Escape the `body` part of .chm source file to 7-bit ASCII, to fix visual +effect on some MBCS Windows systems. + +https://bugs.python.org/issue32174 +""" + +import re +from html.entities import codepoint2name + +# escape the characters which codepoint > 0x7F +def _process(string): + def escape(matchobj): + codepoint = ord(matchobj.group(0)) + + name = codepoint2name.get(codepoint) + if name is None: + return '&#%d;' % codepoint + else: + return '&%s;' % name + + return re.sub(r'[^\x00-\x7F]', escape, string) + +def escape_for_chm(app, pagename, templatename, context, doctree): + # only works for .chm output + if not hasattr(app.builder, 'name') or app.builder.name != 'htmlhelp': + return + + # escape the `body` part to 7-bit ASCII + body = context.get('body') + if body is not None: + context['body'] = _process(body) + +def setup(app): + # `html-page-context` event emitted when the HTML builder has + # created a context dictionary to render a template with. + app.connect('html-page-context', escape_for_chm) + + return {'version': '1.0', 'parallel_read_safe': True} diff -Nru python3.7-3.7.0/Doc/tools/extensions/pyspecific.py python3.7-3.7.1/Doc/tools/extensions/pyspecific.py --- python3.7-3.7.0/Doc/tools/extensions/pyspecific.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tools/extensions/pyspecific.py 2018-10-20 06:04:19.000000000 +0000 @@ -163,6 +163,13 @@ return ret +class PyAwaitableMixin(object): + def handle_signature(self, sig, signode): + ret = super(PyAwaitableMixin, self).handle_signature(sig, signode) + signode.insert(0, addnodes.desc_annotation('awaitable ', 'awaitable ')) + return ret + + class PyCoroutineFunction(PyCoroutineMixin, PyModulelevel): def run(self): self.name = 'py:function' @@ -175,6 +182,18 @@ return PyClassmember.run(self) +class PyAwaitableFunction(PyAwaitableMixin, PyClassmember): + def run(self): + self.name = 'py:function' + return PyClassmember.run(self) + + +class PyAwaitableMethod(PyAwaitableMixin, PyClassmember): + def run(self): + self.name = 'py:method' + return PyClassmember.run(self) + + class PyAbstractMethod(PyClassmember): def handle_signature(self, sig, signode): @@ -394,6 +413,8 @@ app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod) + app.add_directive_to_domain('py', 'awaitablefunction', PyAwaitableFunction) + app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) return {'version': '1.0', 'parallel_read_safe': True} diff -Nru python3.7-3.7.0/Doc/tools/templates/layout.html python3.7-3.7.1/Doc/tools/templates/layout.html --- python3.7-3.7.0/Doc/tools/templates/layout.html 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tools/templates/layout.html 2018-10-20 06:04:19.000000000 +0000 @@ -101,6 +101,14 @@ {% endif %} {% endif %} + {# custom CSS; used in asyncio docs! #} + {{ super() }} {% endblock %} {% block footer %} diff -Nru python3.7-3.7.0/Doc/tutorial/classes.rst python3.7-3.7.1/Doc/tutorial/classes.rst --- python3.7-3.7.0/Doc/tutorial/classes.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tutorial/classes.rst 2018-10-20 06:04:19.000000000 +0000 @@ -387,8 +387,8 @@ the method's instance object before the first argument. If you still don't understand how methods work, a look at the implementation can -perhaps clarify matters. When an instance attribute is referenced that isn't a -data attribute, its class is searched. If the name denotes a valid class +perhaps clarify matters. When a non-data attribute of an instance is +referenced, the instance's class is searched. If the name denotes a valid class attribute that is a function object, a method object is created by packing (pointers to) the instance object and the function object just found together in an abstract object: this is the method object. When the method object is called diff -Nru python3.7-3.7.0/Doc/tutorial/datastructures.rst python3.7-3.7.1/Doc/tutorial/datastructures.rst --- python3.7-3.7.0/Doc/tutorial/datastructures.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tutorial/datastructures.rst 2018-10-20 06:04:19.000000000 +0000 @@ -40,8 +40,8 @@ .. method:: list.remove(x) :noindex: - Remove the first item from the list whose value is equal to *x*. It is an error if - there is no such item. + Remove the first item from the list whose value is equal to *x*. It raises a + ``ValueError`` if there is no such item. .. method:: list.pop([i]) diff -Nru python3.7-3.7.0/Doc/tutorial/inputoutput.rst python3.7-3.7.1/Doc/tutorial/inputoutput.rst --- python3.7-3.7.0/Doc/tutorial/inputoutput.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tutorial/inputoutput.rst 2018-10-20 06:04:19.000000000 +0000 @@ -20,20 +20,41 @@ See the Library Reference for more information on this.) Often you'll want more control over the formatting of your output than simply -printing space-separated values. There are two ways to format your output; the -first way is to do all the string handling yourself; using string slicing and -concatenation operations you can create any layout you can imagine. The -string type has some methods that perform useful operations for padding -strings to a given column width; these will be discussed shortly. The second -way is to use :ref:`formatted string literals `, or the -:meth:`str.format` method. - -The :mod:`string` module contains a :class:`~string.Template` class which offers -yet another way to substitute values into strings. - -One question remains, of course: how do you convert values to strings? Luckily, -Python has ways to convert any value to a string: pass it to the :func:`repr` -or :func:`str` functions. +printing space-separated values. There are several ways to format output. + +* To use :ref:`formatted string literals `, begin a string + with ``f`` or ``F`` before the opening quotation mark or triple quotation mark. + Inside this string, you can write a Python expression between ``{`` and ``}`` + characters that can refer to variables or literal values. + + :: + + >>> year = 2016 + >>> event = 'Referendum' + >>> f'Results of the {year} {event}' + 'Results of the 2016 Referendum' + +* The :meth:`str.format` method of strings requires more manual + effort. You'll still use ``{`` and ``}`` to mark where a variable + will be substituted and can provide detailed formatting directives, + but you'll also need to provide the information to be formatted. + + :: + + >>> yes_votes = 42_572_654 + >>> no_votes = 43_132_495 + >>> percentage = yes_votes / (yes_votes + no_votes) + >>> '{:-9} YES votes {:2.2%}'.format(yes_votes, percentage) + ' 42572654 YES votes 49.67%' + +* Finally, you can do all the string handling yourself by using string slicing and + concatenation operations to create any layout you can imagine. The + string type has some methods that perform useful operations for padding + strings to a given column width. + +When you don't need fancy output but just want a quick display of some +variables for debugging purposes, you can convert any value to a string with +the :func:`repr` or :func:`str` functions. The :func:`str` function is meant to return representations of values which are fairly human-readable, while :func:`repr` is meant to generate representations @@ -67,60 +88,58 @@ ... repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -Here are two ways to write a table of squares and cubes:: +The :mod:`string` module contains a :class:`~string.Template` class that offers +yet another way to substitute values into strings, using placeholders like +``$x`` and replacing them with values from a dictionary, but offers much less +control of the formatting. - >>> for x in range(1, 11): - ... print(repr(x).rjust(2), repr(x*x).rjust(3), end=' ') - ... # Note use of 'end' on previous line - ... print(repr(x*x*x).rjust(4)) - ... - 1 1 1 - 2 4 8 - 3 9 27 - 4 16 64 - 5 25 125 - 6 36 216 - 7 49 343 - 8 64 512 - 9 81 729 - 10 100 1000 - >>> for x in range(1, 11): - ... print('{0:2d} {1:3d} {2:4d}'.format(x, x*x, x*x*x)) +.. _tut-f-strings: + +Formatted String Literals +------------------------- + +:ref:`Formatted string literals ` (also called f-strings for +short) let you include the value of Python expressions inside a string by +prefixing the string with ``f`` or ``F`` and writing expressions as +``{expression}``. + +An optional format specifier can follow the expression. This allows greater +control over how the value is formatted. The following example rounds pi to +three places after the decimal:: + + >>> import math + >>> print(f'The value of pi is approximately {math.pi:.3f}.') + The value of pi is approximately 3.142. + +Passing an integer after the ``':'`` will cause that field to be a minimum +number of characters wide. This is useful for making columns line up. :: + + >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 7678} + >>> for name, phone in table.items(): + ... print(f'{name:10} ==> {phone:10d}') ... - 1 1 1 - 2 4 8 - 3 9 27 - 4 16 64 - 5 25 125 - 6 36 216 - 7 49 343 - 8 64 512 - 9 81 729 - 10 100 1000 + Sjoerd ==> 4127 + Jack ==> 4098 + Dcab ==> 7678 + +Other modifiers can be used to convert the value before it is formatted. +``'!a'`` applies :func:`ascii`, ``'!s'`` applies :func:`str`, and ``'!r'`` +applies :func:`repr`:: -(Note that in the first example, one space between each column was added by the -way :func:`print` works: by default it adds spaces between its arguments.) + >>> animals = 'eels' + >>> print(f'My hovercraft is full of {animals}.') + My hovercraft is full of eels. + >>> print(f'My hovercraft is full of {animals!r}.') + My hovercraft is full of 'eels'. -This example demonstrates the :meth:`str.rjust` method of string -objects, which right-justifies a string in a field of a given width by padding -it with spaces on the left. There are similar methods :meth:`str.ljust` and -:meth:`str.center`. These methods do not write anything, they just return a -new string. If the input string is too long, they don't truncate it, but -return it unchanged; this will mess up your column lay-out but that's usually -better than the alternative, which would be lying about a value. (If you -really want truncation you can always add a slice operation, as in -``x.ljust(n)[:n]``.) +For a reference on these format specifications, see +the reference guide for the :ref:`formatspec`. -There is another method, :meth:`str.zfill`, which pads a numeric string on the -left with zeros. It understands about plus and minus signs:: +.. _tut-string-format: - >>> '12'.zfill(5) - '00012' - >>> '-3.14'.zfill(7) - '-003.14' - >>> '3.14159265359'.zfill(5) - '3.14159265359' +The String format() Method +-------------------------- Basic usage of the :meth:`str.format` method looks like this:: @@ -150,34 +169,6 @@ other='Georg')) The story of Bill, Manfred, and Georg. -``'!a'`` (apply :func:`ascii`), ``'!s'`` (apply :func:`str`) and ``'!r'`` -(apply :func:`repr`) can be used to convert the value before it is formatted:: - - >>> contents = 'eels' - >>> print('My hovercraft is full of {}.'.format(contents)) - My hovercraft is full of eels. - >>> print('My hovercraft is full of {!r}.'.format(contents)) - My hovercraft is full of 'eels'. - -An optional ``':'`` and format specifier can follow the field name. This allows -greater control over how the value is formatted. The following example -rounds Pi to three places after the decimal. - - >>> import math - >>> print('The value of PI is approximately {0:.3f}.'.format(math.pi)) - The value of PI is approximately 3.142. - -Passing an integer after the ``':'`` will cause that field to be a minimum -number of characters wide. This is useful for making tables pretty. :: - - >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 7678} - >>> for name, phone in table.items(): - ... print('{0:10} ==> {1:10d}'.format(name, phone)) - ... - Jack ==> 4098 - Dcab ==> 7678 - Sjoerd ==> 4127 - If you have a really long format string that you don't want to split up, it would be nice if you could reference the variables to be formatted by name instead of by position. This can be done by simply passing the dict and using @@ -198,10 +189,71 @@ This is particularly useful in combination with the built-in function :func:`vars`, which returns a dictionary containing all local variables. +As an example, the following lines produce a tidily-aligned +set of columns giving integers and their squares and cubes:: + + >>> for x in range(1, 11): + ... print('{0:2d} {1:3d} {2:4d}'.format(x, x*x, x*x*x)) + ... + 1 1 1 + 2 4 8 + 3 9 27 + 4 16 64 + 5 25 125 + 6 36 216 + 7 49 343 + 8 64 512 + 9 81 729 + 10 100 1000 + For a complete overview of string formatting with :meth:`str.format`, see :ref:`formatstrings`. +Manual String Formatting +------------------------ + +Here's the same table of squares and cubes, formatted manually:: + + >>> for x in range(1, 11): + ... print(repr(x).rjust(2), repr(x*x).rjust(3), end=' ') + ... # Note use of 'end' on previous line + ... print(repr(x*x*x).rjust(4)) + ... + 1 1 1 + 2 4 8 + 3 9 27 + 4 16 64 + 5 25 125 + 6 36 216 + 7 49 343 + 8 64 512 + 9 81 729 + 10 100 1000 + +(Note that the one space between each column was added by the +way :func:`print` works: it always adds spaces between its arguments.) + +The :meth:`str.rjust` method of string objects right-justifies a string in a +field of a given width by padding it with spaces on the left. There are +similar methods :meth:`str.ljust` and :meth:`str.center`. These methods do +not write anything, they just return a new string. If the input string is too +long, they don't truncate it, but return it unchanged; this will mess up your +column lay-out but that's usually better than the alternative, which would be +lying about a value. (If you really want truncation you can always add a +slice operation, as in ``x.ljust(n)[:n]``.) + +There is another method, :meth:`str.zfill`, which pads a numeric string on the +left with zeros. It understands about plus and minus signs:: + + >>> '12'.zfill(5) + '00012' + >>> '-3.14'.zfill(7) + '-003.14' + >>> '3.14159265359'.zfill(5) + '3.14159265359' + + Old string formatting --------------------- @@ -211,8 +263,8 @@ operation. For example:: >>> import math - >>> print('The value of PI is approximately %5.3f.' % math.pi) - The value of PI is approximately 3.142. + >>> print('The value of pi is approximately %5.3f.' % math.pi) + The value of pi is approximately 3.142. More information can be found in the :ref:`old-string-formatting` section. @@ -289,7 +341,7 @@ >>> f.read() Traceback (most recent call last): File "", line 1, in - ValueError: I/O operation on closed file + ValueError: I/O operation on closed file. .. _tut-filemethods: diff -Nru python3.7-3.7.0/Doc/tutorial/interpreter.rst python3.7-3.7.1/Doc/tutorial/interpreter.rst --- python3.7-3.7.0/Doc/tutorial/interpreter.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tutorial/interpreter.rst 2018-10-20 06:04:19.000000000 +0000 @@ -24,11 +24,11 @@ popular alternative location.) On Windows machines, the Python installation is usually placed in -:file:`C:\\Python36`, though you can change this when you're running the +:file:`C:\\Python37`, though you can change this when you're running the installer. To add this directory to your path, you can type the following command into the command prompt in a DOS box:: - set path=%path%;C:\python36 + set path=%path%;C:\python37 Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on Windows) at the primary prompt causes the interpreter to exit with a zero exit diff -Nru python3.7-3.7.0/Doc/tutorial/introduction.rst python3.7-3.7.1/Doc/tutorial/introduction.rst --- python3.7-3.7.0/Doc/tutorial/introduction.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tutorial/introduction.rst 2018-10-20 06:04:19.000000000 +0000 @@ -143,12 +143,12 @@ "doesn't" >>> "doesn't" # ...or use double quotes instead "doesn't" - >>> '"Yes," he said.' - '"Yes," he said.' - >>> "\"Yes,\" he said." - '"Yes," he said.' - >>> '"Isn\'t," she said.' - '"Isn\'t," she said.' + >>> '"Yes," they said.' + '"Yes," they said.' + >>> "\"Yes,\" they said." + '"Yes," they said.' + >>> '"Isn\'t," they said.' + '"Isn\'t," they said.' In the interactive interpreter, the output string is enclosed in quotes and special characters are escaped with backslashes. While this might sometimes @@ -159,10 +159,10 @@ readable output, by omitting the enclosing quotes and by printing escaped and special characters:: - >>> '"Isn\'t," she said.' - '"Isn\'t," she said.' - >>> print('"Isn\'t," she said.') - "Isn't," she said. + >>> '"Isn\'t," they said.' + '"Isn\'t," they said.' + >>> print('"Isn\'t," they said.') + "Isn't," they said. >>> s = 'First line.\nSecond line.' # \n means newline >>> s # without print(), \n is included in the output 'First line.\nSecond line.' @@ -223,10 +223,14 @@ >>> prefix = 'Py' >>> prefix 'thon' # can't concatenate a variable and a string literal - ... + File "", line 1 + prefix 'thon' + ^ SyntaxError: invalid syntax >>> ('un' * 3) 'ium' - ... + File "", line 1 + ('un' * 3) 'ium' + ^ SyntaxError: invalid syntax If you want to concatenate variables or a variable and a literal, use ``+``:: @@ -320,10 +324,12 @@ Therefore, assigning to an indexed position in the string results in an error:: >>> word[0] = 'J' - ... + Traceback (most recent call last): + File "", line 1, in TypeError: 'str' object does not support item assignment >>> word[2:] = 'py' - ... + Traceback (most recent call last): + File "", line 1, in TypeError: 'str' object does not support item assignment If you need a different string, you should create a new one:: diff -Nru python3.7-3.7.0/Doc/tutorial/modules.rst python3.7-3.7.1/Doc/tutorial/modules.rst --- python3.7-3.7.0/Doc/tutorial/modules.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/tutorial/modules.rst 2018-10-20 06:04:19.000000000 +0000 @@ -249,7 +249,7 @@ directory. * There is more detail on this process, including a flow chart of the - decisions, in PEP 3147. + decisions, in :pep:`3147`. .. _tut-standardmodules: diff -Nru python3.7-3.7.0/Doc/using/cmdline.rst python3.7-3.7.1/Doc/using/cmdline.rst --- python3.7-3.7.0/Doc/using/cmdline.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/using/cmdline.rst 2018-10-20 06:04:19.000000000 +0000 @@ -182,13 +182,13 @@ .. code-block:: none - Python 3.6.0b2+ + Python 3.7.0b2+ When given twice, print more information about the build, like: .. code-block:: none - Python 3.6.0b2+ (3.6:84a3c5003510+, Oct 26 2016, 02:33:55) + Python 3.7.0b2+ (3.7:0c076caaa8, Sep 22 2018, 12:04:24) [GCC 6.2.0 20161005] .. versionadded:: 3.6 @@ -889,7 +889,7 @@ If this environment variable is not set at all, then the interpreter defaults to using the current locale settings, *unless* the current locale is identified as a legacy ASCII-based locale - (as descibed for :envvar:`PYTHONCOERCECLOCALE`), and locale coercion is + (as described for :envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or fails. In such legacy locales, the interpreter will default to enabling UTF-8 mode unless explicitly instructed not to do so. diff -Nru python3.7-3.7.0/Doc/using/venv-create.inc python3.7-3.7.1/Doc/using/venv-create.inc --- python3.7-3.7.0/Doc/using/venv-create.inc 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/using/venv-create.inc 2018-10-20 06:04:19.000000000 +0000 @@ -20,11 +20,6 @@ .. versionchanged:: 3.5 The use of ``venv`` is now recommended for creating virtual environments. -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - `__ - .. highlight:: none On Windows, invoke the ``venv`` command as follows:: diff -Nru python3.7-3.7.0/Doc/using/windows.rst python3.7-3.7.1/Doc/using/windows.rst --- python3.7-3.7.0/Doc/using/windows.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/using/windows.rst 2018-10-20 06:04:19.000000000 +0000 @@ -147,7 +147,7 @@ | | ``.pyc``. | | +---------------------------+--------------------------------------+--------------------------+ | PrependPath | Add install and Scripts directories | 0 | -| | tho :envvar:`PATH` and ``.PY`` to | | +| | to :envvar:`PATH` and ``.PY`` to | | | | :envvar:`PATHEXT` | | +---------------------------+--------------------------------------+--------------------------+ | Shortcuts | Create shortcuts for the interpreter,| 1 | @@ -193,13 +193,13 @@ For example, to silently install a default, system-wide Python installation, you could use the following command (from an elevated command prompt):: - python-3.6.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 + python-3.7.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 To allow users to easily install a personal copy of Python without the test suite, you could provide a shortcut with the following command. This will display a simplified initial page and disallow customization:: - python-3.6.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0 + python-3.7.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0 SimpleInstall=1 SimpleInstallDescription="Just for me, no test suite." (Note that omitting the launcher also omits file associations, and is only @@ -210,7 +210,7 @@ alongside the executable. This file specifies a list of options and values. When a value is provided as an attribute, it will be converted to a number if possible. Values provided as element text are always left as strings. This -example file sets the same options and the previous example: +example file sets the same options as the previous example: .. code-block:: xml @@ -236,13 +236,13 @@ useful to have a locally cached copy. Execute the following command from Command Prompt to download all possible -required files. Remember to substitute ``python-3.6.0.exe`` for the actual +required files. Remember to substitute ``python-3.7.0.exe`` for the actual name of your installer, and to create layouts in their own directories to avoid collisions between files with the same name. :: - python-3.6.0.exe /layout [optional target directory] + python-3.7.0.exe /layout [optional target directory] You may also specify the ``/quiet`` option to hide the progress display. @@ -349,7 +349,7 @@ .. code-block:: doscon - C:\>set PATH=C:\Program Files\Python 3.6;%PATH% + C:\>set PATH=C:\Program Files\Python 3.7;%PATH% C:\>set PYTHONPATH=%PYTHONPATH%;C:\My_python_lib C:\>python @@ -381,7 +381,7 @@ .. seealso:: - https://support.microsoft.com/en-us/help/100843/environment-variables-in-windows-nt + https://www.microsoft.com/en-us/wdsi/help/folder-variables Environment variables in Windows NT https://technet.microsoft.com/en-us/library/cc754250.aspx @@ -422,7 +422,7 @@ example variable could look like this (assuming the first two entries already existed):: - C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Python 3.6 + C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Python 3.7 .. _launcher: @@ -610,7 +610,7 @@ Two .ini files will be searched by the launcher - ``py.ini`` in the current user's "application data" directory (i.e. the directory returned by calling the -Windows function SHGetFolderPath with CSIDL_LOCAL_APPDATA) and ``py.ini`` in the +Windows function ``SHGetFolderPath`` with ``CSIDL_LOCAL_APPDATA``) and ``py.ini`` in the same directory as the launcher. The same .ini files are used for both the 'console' version of the launcher (i.e. py.exe) and for the 'windows' version (i.e. pyw.exe) @@ -816,7 +816,7 @@ These will ensure that the files in a system-wide installation will not take precedence over the copy of the standard library bundled with your application. Otherwise, your users may experience problems using your application. Note that -the first suggestion is the best, as the other may still be susceptible to +the first suggestion is the best, as the others may still be susceptible to non-standard paths in the registry and user site-packages. .. versionchanged:: diff -Nru python3.7-3.7.0/Doc/whatsnew/3.3.rst python3.7-3.7.1/Doc/whatsnew/3.3.rst --- python3.7-3.7.0/Doc/whatsnew/3.3.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/whatsnew/3.3.rst 2018-10-20 06:04:19.000000000 +0000 @@ -303,7 +303,7 @@ application. Running ``py`` follows the same version selection rules as implicitly launching scripts, but a more specific version can be selected by passing appropriate arguments (such as ``-3`` to request Python 3 when -Python 2 is also installed, or ``-2.6`` to specifclly request an earlier +Python 2 is also installed, or ``-2.6`` to specifically request an earlier Python version when a more recent version is installed). In addition to the launcher, the Windows installer now includes an @@ -1414,7 +1414,7 @@ :class:`http.server.BaseHTTPRequestHandler` now buffers the headers and writes them all at once when :meth:`~http.server.BaseHTTPRequestHandler.end_headers` is called. A new method :meth:`~http.server.BaseHTTPRequestHandler.flush_headers` -can be used to directly manage when the accumlated headers are sent. +can be used to directly manage when the accumulated headers are sent. (Contributed by Andrew Schaaf in :issue:`3709`.) :class:`http.server` now produces valid ``HTML 4.01 strict`` output. @@ -2386,7 +2386,7 @@ finder, you will need to remove keys paired with values of ``None`` **and** :class:`imp.NullImporter` to be backwards-compatible. This will lead to extra overhead on older versions of Python that re-insert ``None`` into - :attr:`sys.path_importer_cache` where it repesents the use of implicit + :attr:`sys.path_importer_cache` where it represents the use of implicit finders, but semantically it should not change anything. * :class:`importlib.abc.Finder` no longer specifies a `find_module()` abstract diff -Nru python3.7-3.7.0/Doc/whatsnew/3.5.rst python3.7-3.7.1/Doc/whatsnew/3.5.rst --- python3.7-3.7.0/Doc/whatsnew/3.5.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/whatsnew/3.5.rst 2018-10-20 06:04:19.000000000 +0000 @@ -777,18 +777,18 @@ Notable changes in the :mod:`asyncio` module since Python 3.4.0: -* New debugging APIs: :meth:`loop.set_debug() ` - and :meth:`loop.get_debug() ` methods. +* New debugging APIs: :meth:`loop.set_debug() ` + and :meth:`loop.get_debug() ` methods. (Contributed by Victor Stinner.) * The proactor event loop now supports SSL. (Contributed by Antoine Pitrou and Victor Stinner in :issue:`22560`.) -* A new :meth:`loop.is_closed() ` method to +* A new :meth:`loop.is_closed() ` method to check if the event loop is closed. (Contributed by Victor Stinner in :issue:`21326`.) -* A new :meth:`loop.create_task() ` +* A new :meth:`loop.create_task() ` to conveniently create and schedule a new :class:`~asyncio.Task` for a coroutine. The ``create_task`` method is also used by all asyncio functions that wrap coroutines into tasks, such as @@ -805,10 +805,10 @@ (Contributed by Yury Selivanov.) * New :meth:`loop.set_task_factory() - ` and - :meth:`loop.get_task_factory() ` + ` and + :meth:`loop.get_task_factory() ` methods to customize the task factory that :meth:`loop.create_task() - ` method uses. (Contributed by Yury + ` method uses. (Contributed by Yury Selivanov.) * New :meth:`Queue.join() ` and @@ -822,7 +822,7 @@ Updates in 3.5.1: * The :func:`~asyncio.ensure_future` function and all functions that - use it, such as :meth:`loop.run_until_complete() `, + use it, such as :meth:`loop.run_until_complete() `, now accept all kinds of :term:`awaitable objects `. (Contributed by Yury Selivanov.) @@ -834,20 +834,20 @@ method to check if the transport is closing or closed. (Contributed by Yury Selivanov.) -* The :meth:`loop.create_server() ` +* The :meth:`loop.create_server() ` method can now accept a list of hosts. (Contributed by Yann Sionneau.) Updates in 3.5.2: -* New :meth:`loop.create_future() ` +* New :meth:`loop.create_future() ` method to create Future objects. This allows alternative event loop implementations, such as `uvloop `_, to provide a faster :class:`asyncio.Future` implementation. (Contributed by Yury Selivanov.) -* New :meth:`loop.get_exception_handler() ` +* New :meth:`loop.get_exception_handler() ` method to get the current exception handler. (Contributed by Yury Selivanov.) @@ -856,13 +856,13 @@ sequence appears. (Contributed by Mark Korenberg.) -* The :meth:`loop.create_connection() ` - and :meth:`loop.create_server() ` +* The :meth:`loop.create_connection() ` + and :meth:`loop.create_server() ` methods are optimized to avoid calling the system ``getaddrinfo`` function if the address is already resolved. (Contributed by A. Jesse Jiryu Davis.) -* The :meth:`loop.sock_connect(sock, address) ` +* The :meth:`loop.sock_connect(sock, address) ` no longer requires the *address* to be resolved prior to the call. (Contributed by A. Jesse Jiryu Davis.) @@ -2471,7 +2471,7 @@ parameter to help control the ``opt-`` tag. Because of this, the *debug_override* parameter of the function is now deprecated. `.pyo` files are also no longer supported as a file argument to the Python interpreter and - thus serve no purpose when distributed on their own (i.e. sourcless code + thus serve no purpose when distributed on their own (i.e. sourceless code distribution). Due to the fact that the magic number for bytecode has changed in Python 3.5, all old `.pyo` files from previous versions of Python are invalid regardless of this PEP. diff -Nru python3.7-3.7.0/Doc/whatsnew/3.6.rst python3.7-3.7.1/Doc/whatsnew/3.6.rst --- python3.7-3.7.0/Doc/whatsnew/3.6.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/whatsnew/3.6.rst 2018-10-20 06:04:19.000000000 +0000 @@ -824,7 +824,7 @@ (Contributed by Yury Selivanov in :issue:`28613`.) * The :func:`~asyncio.ensure_future` function and all functions that - use it, such as :meth:`loop.run_until_complete() `, + use it, such as :meth:`loop.run_until_complete() `, now accept all kinds of :term:`awaitable objects `. (Contributed by Yury Selivanov.) @@ -836,18 +836,18 @@ method to check if the transport is closing or closed. (Contributed by Yury Selivanov.) -* The :meth:`loop.create_server() ` +* The :meth:`loop.create_server() ` method can now accept a list of hosts. (Contributed by Yann Sionneau.) -* New :meth:`loop.create_future() ` +* New :meth:`loop.create_future() ` method to create Future objects. This allows alternative event loop implementations, such as `uvloop `_, to provide a faster :class:`asyncio.Future` implementation. (Contributed by Yury Selivanov in :issue:`27041`.) -* New :meth:`loop.get_exception_handler() ` +* New :meth:`loop.get_exception_handler() ` method to get the current exception handler. (Contributed by Yury Selivanov in :issue:`27040`.) @@ -860,12 +860,12 @@ has been improved. (Contributed by Mark Korenberg in :issue:`28370`.) -* The :meth:`loop.getaddrinfo() ` +* The :meth:`loop.getaddrinfo() ` method is optimized to avoid calling the system ``getaddrinfo`` function if the address is already resolved. (Contributed by A. Jesse Jiryu Davis.) -* The :meth:`loop.stop() ` +* The :meth:`loop.stop() ` method has been changed to stop the loop immediately after the current iteration. Any new callbacks scheduled as a result of the last iteration will be discarded. @@ -876,7 +876,7 @@ the :exc:`StopIteration` exception. (Contributed by Chris Angelico in :issue:`26221`.) -* New :meth:`loop.connect_accepted_socket() ` +* New :meth:`loop.connect_accepted_socket() ` method to be used by servers that accept connections outside of asyncio, but that use asyncio to handle them. (Contributed by Jim Fulton in :issue:`27392`.) @@ -884,7 +884,7 @@ * ``TCP_NODELAY`` flag is now set for all TCP transports by default. (Contributed by Yury Selivanov in :issue:`27456`.) -* New :meth:`loop.shutdown_asyncgens() ` +* New :meth:`loop.shutdown_asyncgens() ` to properly close pending asynchronous generators before closing the loop. (Contributed by Yury Selivanov in :issue:`28003`.) @@ -1177,13 +1177,22 @@ maxlines. Clicking on a context line jumps the editor to that line. Context colors for custom themes is added to Highlights tab of Settings dialog. (Contributed by Cheryl Sabella and Terry Jan Reedy in :issue:`33642`, -:issue:`33768`, and :issue:`33679`) +:issue:`33768`, and :issue:`33679`.) On Windows, a new API call tells Windows that tk scales for DPI. On Windows 8.1+ or 10, with DPI compatibility properties of the Python binary unchanged, and a monitor resolution greater than 96 DPI, this should make text and lines sharper. It should otherwise have no effect. -(Contributed by Terry Jan Reedy in :issue:`33656`). +(Contributed by Terry Jan Reedy in :issue:`33656`.) + +New in 3.6.7: + +Output over N lines (50 by default) is squeezed down to a button. +N can be changed in the PyShell section of the General page of the +Settings dialog. Fewer, but possibly extra long, lines can be squeezed by +right clicking on the output. Squeezed output can be expanded in place +by double-clicking the button or into the clipboard or a separate window +by right-clicking the button. (Contributed by Tal Einat in :issue:`1529353`.) importlib @@ -1614,7 +1623,7 @@ `_.) A new :const:`~typing.TYPE_CHECKING` constant that is assumed to be -``True`` by the static type chekers, but is ``False`` at runtime. +``True`` by the static type checkers, but is ``False`` at runtime. (Contributed by Guido van Rossum in `Github #230 `_.) @@ -1856,7 +1865,7 @@ For more information, see :pep:`7` and :issue:`17884`. * Cross-compiling CPython with the Android NDK and the Android API level set to - 21 (Android 5.0 Lollilop) or greater runs successfully. While Android is not + 21 (Android 5.0 Lollipop) or greater runs successfully. While Android is not yet a supported platform, the Python test suite runs on the Android emulator with only about 16 tests failures. See the Android meta-issue :issue:`26865`. diff -Nru python3.7-3.7.0/Doc/whatsnew/3.7.rst python3.7-3.7.1/Doc/whatsnew/3.7.rst --- python3.7-3.7.0/Doc/whatsnew/3.7.rst 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Doc/whatsnew/3.7.rst 2018-10-20 06:04:19.000000000 +0000 @@ -320,7 +320,8 @@ ----------------------------------------------------- Python 3.7 allows defining :meth:`__getattr__` on modules and will call -it whenever a module attribute is otherwise not found. +it whenever a module attribute is otherwise not found. Defining +:meth:`__dir__` on modules is now also allowed. A typical example of where this may be useful is module attribute deprecation and lazy loading. @@ -336,8 +337,11 @@ PEP 564: New Time Functions With Nanosecond Resolution ------------------------------------------------------ -:pep:`564` adds six new "nanosecond" variants of existing functions -to the :mod:`time` module: +The resolution of clocks in modern systems can exceed the limited precision +of a floating point number returned by the :func:`time.time` function +and its variants. To avoid loss of precision, :pep:`564` adds six new +"nanosecond" variants of the existing timer functions to the :mod:`time` +module: * :func:`time.clock_gettime_ns` * :func:`time.clock_settime_ns` @@ -346,12 +350,11 @@ * :func:`time.process_time_ns` * :func:`time.time_ns` -The new functions are similar in function to the existing functions -without the ``_ns`` suffix. They differ by returning nanoseconds as -integers instead of fractional seconds. +The new functions return the number of nanoseconds as an integer value. -On Linux and Windows the resolution of :func:`time.time_ns` is 3 times -better than that of :func:`time.time`. +`Measurements `_ +show that on Linux and Windows the resolution of :func:`time.time_ns` is +approximately 3 times better than that of :func:`time.time`. .. seealso:: @@ -418,19 +421,6 @@ PEP written and implemented by Ivan Levkivskyi -.. _whatsnew37-devmode: - -Development Runtime Mode: -X dev --------------------------------- - -The new :option:`-X` ``dev`` command line option or the new -:envvar:`PYTHONDEVMODE` environment variable can be used to enable -CPython's *development mode*. When in development mode, CPython performs -additional runtime checks which are too expensive to be enabled by default. -See :option:`-X` ``dev`` documentation for the full description of the effects -of this mode. - - .. _whatsnew37-pep552: PEP 552: Hash-based .pyc Files @@ -460,6 +450,11 @@ See :ref:`pyc-invalidation` for more information. +.. seealso:: + + :pep:`552` -- Deterministic pycs + PEP written and implemented by Benjamin Peterson + .. _whatsnew37-pep545: @@ -482,6 +477,19 @@ Victor Stinner. +.. _whatsnew37-devmode: + +Development Runtime Mode: -X dev +-------------------------------- + +The new :option:`-X` ``dev`` command line option or the new +:envvar:`PYTHONDEVMODE` environment variable can be used to enable +CPython's *development mode*. When in development mode, CPython performs +additional runtime checks that are too expensive to be enabled by default. +See :option:`-X` ``dev`` documentation for the full description of the effects +of this mode. + + Other Language Changes ====================== @@ -629,10 +637,10 @@ (Contributed by Yury Selivanov in :issue:`32314`.) * asyncio gained support for :mod:`contextvars`. - :meth:`loop.call_soon() `, - :meth:`loop.call_soon_threadsafe() `, - :meth:`loop.call_later() `, - :meth:`loop.call_at() `, and + :meth:`loop.call_soon() `, + :meth:`loop.call_soon_threadsafe() `, + :meth:`loop.call_later() `, + :meth:`loop.call_at() `, and :meth:`Future.add_done_callback() ` have a new optional keyword-only *context* parameter. :class:`Tasks ` now track their context automatically. @@ -643,11 +651,11 @@ to ``asyncio.get_event_loop().create_task()``. (Contributed by Andrew Svetlov in :issue:`32311`.) -* The new :meth:`loop.start_tls() ` +* The new :meth:`loop.start_tls() ` method can be used to upgrade an existing connection to TLS. (Contributed by Yury Selivanov in :issue:`23749`.) -* The new :meth:`loop.sock_recv_into() ` +* The new :meth:`loop.sock_recv_into() ` method allows reading data from a socket directly into a provided buffer making it possible to reduce data copies. (Contributed by Antoine Pitrou in :issue:`31819`.) @@ -675,13 +683,13 @@ can be used to determine if the writer is closing. (Contributed by Andrew Svetlov in :issue:`32391`.) -* The new :meth:`loop.sock_sendfile() ` +* The new :meth:`loop.sock_sendfile() ` coroutine method allows sending files using :mod:`os.sendfile` when possible. (Contributed by Andrew Svetlov in :issue:`32410`.) -* The new :meth:`Task.get_loop() ` and - :meth:`Future.get_loop() ` methods - return the instance of the loop on which a task or a future were created. +* The new :meth:`Future.get_loop() ` and + ``Task.get_loop()`` methods return the instance of the loop on which a task or + a future were created. :meth:`Server.get_loop() ` allows doing the same for :class:`asyncio.Server` objects. (Contributed by Yury Selivanov in :issue:`32415` and @@ -690,8 +698,8 @@ * It is now possible to control how instances of :class:`asyncio.Server` begin serving. Previously, the server would start serving immediately when created. The new *start_serving* keyword argument to - :meth:`loop.create_server() ` and - :meth:`loop.create_unix_server() `, + :meth:`loop.create_server() ` and + :meth:`loop.create_unix_server() `, as well as :meth:`Server.start_serving() `, and :meth:`Server.serve_forever() ` can be used to decouple server instantiation and serving. The new @@ -709,20 +717,20 @@ (Contributed by Yury Selivanov in :issue:`32662`.) * Callback objects returned by - :func:`loop.call_later() ` + :func:`loop.call_later() ` gained the new :meth:`when() ` method which returns an absolute scheduled callback timestamp. (Contributed by Andrew Svetlov in :issue:`32741`.) * The :meth:`loop.create_datagram_endpoint() \ - ` method + ` method gained support for Unix sockets. (Contributed by Quentin Dawans in :issue:`31245`.) * The :func:`asyncio.open_connection`, :func:`asyncio.start_server` functions, - :meth:`loop.create_connection() `, - :meth:`loop.create_server() `, - :meth:`loop.create_accepted_socket() ` + :meth:`loop.create_connection() `, + :meth:`loop.create_server() `, + :meth:`loop.create_accepted_socket() ` methods and their corresponding UNIX socket variants now accept the *ssl_handshake_timeout* keyword argument. (Contributed by Neil Aspinall in :issue:`29970`.) @@ -990,13 +998,22 @@ maxlines. Clicking on a context line jumps the editor to that line. Context colors for custom themes is added to Highlights tab of Settings dialog. (Contributed by Cheryl Sabella and Terry Jan Reedy in :issue:`33642`, -:issue:`33768`, and :issue:`33679`, +:issue:`33768`, and :issue:`33679`.) On Windows, a new API call tells Windows that tk scales for DPI. On Windows 8.1+ or 10, with DPI compatibility properties of the Python binary unchanged, and a monitor resolution greater than 96 DPI, this should make text and lines sharper. It should otherwise have no effect. -(Contributed by Terry Jan Reedy in :issue:`33656`). +(Contributed by Terry Jan Reedy in :issue:`33656`.) + +New in 3.7.1: + +Output over N lines (50 by default) is squeezed down to a button. +N can be changed in the PyShell section of the General page of the +Settings dialog. Fewer, but possibly extra long, lines can be squeezed by +right clicking on the output. Squeezed output can be expanded in place +by double-clicking the button or into the clipboard or a separate window +by right-clicking the button. (Contributed by Tal Einat in :issue:`1529353`.) The changes above have been backported to 3.6 maintenance releases. @@ -1296,8 +1313,8 @@ .. note:: The improved host name check requires a *libssl* implementation compatible with OpenSSL 1.0.2 or 1.1. Consequently, OpenSSL 0.9.8 and 1.0.1 are no - longer supported. The ssl module is mostly compatible with LibreSSL 2.7.2 - and newer. + longer supported (see :ref:`37-platform-support-removals` for more details). + The ssl module is mostly compatible with LibreSSL 2.7.2 and newer. The ``ssl`` module no longer sends IP addresses in SNI TLS extension. (Contributed by Christian Heimes in :issue:`32185`.) @@ -1338,10 +1355,14 @@ OpenSSL 1.1 APIs for setting the minimum and maximum TLS protocol version are available as :attr:`SSLContext.minimum_version ` and :attr:`SSLContext.maximum_version `. -Supported protocols are indicated by serveral new flags, such as +Supported protocols are indicated by several new flags, such as :data:`~ssl.HAS_TLSv1_1`. (Contributed by Christian Heimes in :issue:`32609`.) +Added :attr:`SSLContext.post_handshake_auth` to enable and +:meth:`ssl.SSLSocket.verify_client_post_handshake` to initiate TLS 1.3 +post-handshake authentication. +(Contributed by Christian Heimes in :issue:`34670`.) string ------ @@ -1561,6 +1582,15 @@ (Contributed by Nick Coghlan in :issue:`31975`.) +xml +--- + +As mitigation against DTD and external entity retrieval, the +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. +(Contributed by Christian Heimes in :issue:`17239`.) + + xml.etree --------- @@ -1915,8 +1945,7 @@ Support for directly ``await``-ing instances of :class:`asyncio.Lock` and other asyncio synchronization primitives has been deprecated. An asynchronous context manager must be used in order to acquire and release -the synchronization resource. See :ref:`async-with-locks` for more -information. +the synchronization resource. (Contributed by Andrew Svetlov in :issue:`32253`.) The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` @@ -2007,7 +2036,7 @@ socket ------ -The silent argument value trunctation in :func:`socket.htons` and +The silent argument value truncation in :func:`socket.htons` and :func:`socket.ntohs` has been deprecated. In future versions of Python, if the passed argument is larger than 16 bits, an exception will be raised. (Contributed by Oren Milman in :issue:`28332`.) @@ -2061,10 +2090,33 @@ (Contributed by Antoine Pitrou in :issue:`16500`.) +.. _37-platform-support-removals: + Platform Support Removals ========================= -FreeBSD 9 and older are no longer officially supported. +* FreeBSD 9 and older are no longer officially supported. +* For full Unicode support, including within extension modules, \*nix platforms + are now expected to provide at least one of ``C.UTF-8`` (full locale), + ``C.utf8`` (full locale) or ``UTF-8`` (``LC_CTYPE``-only locale) as an + alternative to the legacy ``ASCII``-based ``C`` locale. +* OpenSSL 0.9.8 and 1.0.1 are no longer supported, which means building CPython + 3.7 with SSL/TLS support on older platforms still using these versions + requires custom build options that link to a more recent version of OpenSSL. + + Notably, this issue affects the Debian 8 (aka "jessie") and Ubuntu 14.04 + (aka "Trusty") LTS Linux distributions, as they still use OpenSSL 1.0.1 by + default. + + Debian 9 ("stretch") and Ubuntu 16.04 ("xenial"), as well as recent releases + of other LTS Linux releases (e.g. RHEL/CentOS 7.5, SLES 12-SP3), use OpenSSL + 1.0.2 or later, and remain supported in the default build configuration. + + CPython's own :source:`CI configuration file <.travis.yml>` provides an + example of using the SSL + :source:`compatibility testing infrastructure ` in + CPython's test suite to build and link against OpenSSL 1.1.0 rather than an + outdated system provided OpenSSL. API and Feature Removals @@ -2262,7 +2314,7 @@ * Because :func:`shutil.rmtree` is now implemented using the :func:`os.scandir` function, the user specified handler *onerror* is now called with the first - argument ``os.scandir`` instead of ``os.listdir`` when listing the direcory + argument ``os.scandir`` instead of ``os.listdir`` when listing the directory is failed. * Support for nested sets and set operations in regular expressions as in @@ -2329,11 +2381,11 @@ (Contributed by Brett Cannon in :issue:`33169`.) * In :mod:`asyncio`, - :meth:`loop.sock_recv() `, - :meth:`loop.sock_sendall() `, - :meth:`loop.sock_accept() `, - :meth:`loop.getaddrinfo() `, - :meth:`loop.getnameinfo() ` + :meth:`loop.sock_recv() `, + :meth:`loop.sock_sendall() `, + :meth:`loop.sock_accept() `, + :meth:`loop.getaddrinfo() `, + :meth:`loop.getnameinfo() ` have been changed to be proper coroutine methods to match their documentation. Previously, these methods returned :class:`asyncio.Future` instances. @@ -2375,6 +2427,12 @@ dictionary are now being implicitly converted to strings. (Contributed by James Tocknell in :issue:`23835`.) +* Several undocumented internal imports were removed. + One example is that ``os.errno`` is no longer available; use ``import errno`` + directly instead. + Note that such undocumented internal imports may be removed any time without + notice, even in micro version releases. + Changes in the C API -------------------- @@ -2448,3 +2506,20 @@ This simplified the interpreter and fixed a couple of obscure bugs caused by having swap exception state when entering or exiting a generator. (Contributed by Mark Shannon in :issue:`25612`.) + +Notable changes in Python 3.7.1 +=============================== + +Starting in 3.7.1, :c:func:`Py_Initialize` now consistently reads and respects +all of the same environment settings as :c:func:`Py_Main` (in earlier Python +versions, it respected an ill-defined subset of those environment variables, +while in Python 3.7.0 it didn't read any of them due to :issue:`34247`). If +this behavior is unwanted, set :c:data:`Py_IgnoreEnvironmentFlag` to 1 before +calling :c:func:`Py_Initialize`. + +In 3.7.1 the C API for Context Variables +:ref:`was updated ` to use +:c:type:`PyObject` pointers. See also :issue:`34762`. + +:mod:`xml.dom.minidom` and :mod:`xml.sax` modules no longer process +external entities by default. See also :issue:`17239`. diff -Nru python3.7-3.7.0/Include/context.h python3.7-3.7.1/Include/context.h --- python3.7-3.7.0/Include/context.h 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Include/context.h 2018-10-20 06:04:19.000000000 +0000 @@ -22,19 +22,19 @@ #define PyContextToken_CheckExact(o) (Py_TYPE(o) == &PyContextToken_Type) -PyAPI_FUNC(PyContext *) PyContext_New(void); -PyAPI_FUNC(PyContext *) PyContext_Copy(PyContext *); -PyAPI_FUNC(PyContext *) PyContext_CopyCurrent(void); +PyAPI_FUNC(PyObject *) PyContext_New(void); +PyAPI_FUNC(PyObject *) PyContext_Copy(PyObject *); +PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); -PyAPI_FUNC(int) PyContext_Enter(PyContext *); -PyAPI_FUNC(int) PyContext_Exit(PyContext *); +PyAPI_FUNC(int) PyContext_Enter(PyObject *); +PyAPI_FUNC(int) PyContext_Exit(PyObject *); /* Create a new context variable. default_value can be NULL. */ -PyAPI_FUNC(PyContextVar *) PyContextVar_New( +PyAPI_FUNC(PyObject *) PyContextVar_New( const char *name, PyObject *default_value); @@ -54,21 +54,19 @@ '*value' will be a new ref, if not NULL. */ PyAPI_FUNC(int) PyContextVar_Get( - PyContextVar *var, PyObject *default_value, PyObject **value); + PyObject *var, PyObject *default_value, PyObject **value); /* Set a new value for the variable. Returns NULL if an error occurs. */ -PyAPI_FUNC(PyContextToken *) PyContextVar_Set( - PyContextVar *var, PyObject *value); +PyAPI_FUNC(PyObject *) PyContextVar_Set(PyObject *var, PyObject *value); /* Reset a variable to its previous value. Returns 0 on success, -1 on error. */ -PyAPI_FUNC(int) PyContextVar_Reset( - PyContextVar *var, PyContextToken *token); +PyAPI_FUNC(int) PyContextVar_Reset(PyObject *var, PyObject *token); /* This method is exposed only for CPython tests. Don not use it. */ diff -Nru python3.7-3.7.0/Include/import.h python3.7-3.7.1/Include/import.h --- python3.7-3.7.0/Include/import.h 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Include/import.h 2018-10-20 06:04:19.000000000 +0000 @@ -54,9 +54,6 @@ PyObject *name ); #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *, PyObject *); -#endif PyAPI_FUNC(PyObject *) PyImport_AddModule( const char *name /* UTF-8 encoded string */ ); diff -Nru python3.7-3.7.0/Include/patchlevel.h python3.7-3.7.1/Include/patchlevel.h --- python3.7-3.7.0/Include/patchlevel.h 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Include/patchlevel.h 2018-10-20 06:04:19.000000000 +0000 @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 0 +#define PY_MICRO_VERSION 1 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.7.0" +#define PY_VERSION "3.7.1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff -Nru python3.7-3.7.0/Include/pyexpat.h python3.7-3.7.1/Include/pyexpat.h --- python3.7-3.7.0/Include/pyexpat.h 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Include/pyexpat.h 2018-10-20 06:04:19.000000000 +0000 @@ -3,7 +3,7 @@ /* note: you must import expat.h before importing this module! */ -#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.0" +#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.1" #define PyExpat_CAPSULE_NAME "pyexpat.expat_CAPI" struct PyExpat_CAPI @@ -48,6 +48,8 @@ enum XML_Status (*SetEncoding)(XML_Parser parser, const XML_Char *encoding); int (*DefaultUnknownEncodingHandler)( void *encodingHandlerData, const XML_Char *name, XML_Encoding *info); + /* might be none for expat < 2.1.0 */ + int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); /* always add new stuff to the end! */ }; diff -Nru python3.7-3.7.0/Include/pylifecycle.h python3.7-3.7.1/Include/pylifecycle.h --- python3.7-3.7.0/Include/pylifecycle.h 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Include/pylifecycle.h 2018-10-20 06:04:19.000000000 +0000 @@ -51,14 +51,24 @@ const char *errors); /* PEP 432 Multi-phase initialization API (Private while provisional!) */ -PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *); +PyAPI_FUNC(_PyInitError) _Py_InitializeCore( + PyInterpreterState **interp_p, + const _PyCoreConfig *config); PyAPI_FUNC(int) _Py_IsCoreInitialized(void); +PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig( + const _PyCoreConfig *config); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(void) _Py_Initialize_ReadEnvVarsNoAlloc(void); +#endif PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *); PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); PyAPI_FUNC(int) _PyCoreConfig_Copy( _PyCoreConfig *config, const _PyCoreConfig *config2); +PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig( + const _PyCoreConfig *config); + PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read( _PyMainInterpreterConfig *config, @@ -68,14 +78,16 @@ _PyMainInterpreterConfig *config, const _PyMainInterpreterConfig *config2); -PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *); -#endif +PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter( + PyInterpreterState *interp, + const _PyMainInterpreterConfig *config); +#endif /* !defined(Py_LIMITED_API) */ + /* Initialization and finalization */ PyAPI_FUNC(void) Py_Initialize(void); PyAPI_FUNC(void) Py_InitializeEx(int); #ifndef Py_LIMITED_API -PyAPI_FUNC(_PyInitError) _Py_InitializeEx_Private(int, int); PyAPI_FUNC(void) _Py_FatalInitError(_PyInitError err) _Py_NO_RETURN; #endif PyAPI_FUNC(void) Py_Finalize(void); diff -Nru python3.7-3.7.0/Include/pystate.h python3.7-3.7.1/Include/pystate.h --- python3.7-3.7.0/Include/pystate.h 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Include/pystate.h 2018-10-20 06:04:19.000000000 +0000 @@ -79,8 +79,11 @@ #define _PyCoreConfig_INIT \ (_PyCoreConfig){ \ .install_signal_handlers = -1, \ + .ignore_environment = -1, \ .use_hash_seed = -1, \ .coerce_c_locale = -1, \ + .faulthandler = -1, \ + .tracemalloc = -1, \ .utf8_mode = -1, \ .argc = -1, \ .nmodule_search_path = -1} diff -Nru python3.7-3.7.0/Lib/antigravity.py python3.7-3.7.1/Lib/antigravity.py --- python3.7-3.7.0/Lib/antigravity.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/antigravity.py 2018-10-20 06:04:19.000000000 +0000 @@ -11,7 +11,7 @@ 37.857713 -122.544543 ''' - # http://xkcd.com/426/ + # https://xkcd.com/426/ h = hashlib.md5(datedow).hexdigest() p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])] print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:])) diff -Nru python3.7-3.7.0/Lib/asyncio/base_events.py python3.7-3.7.1/Lib/asyncio/base_events.py --- python3.7-3.7.0/Lib/asyncio/base_events.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/asyncio/base_events.py 2018-10-20 06:04:19.000000000 +0000 @@ -61,6 +61,11 @@ _FATAL_ERROR_IGNORE = (BrokenPipeError, ConnectionResetError, ConnectionAbortedError) +_HAS_IPv6 = hasattr(socket, 'AF_INET6') + +# Maximum timeout passed to select to avoid OS limitations +MAXIMUM_SELECT_TIMEOUT = 24 * 3600 + def _format_handle(handle): cb = handle._callback @@ -123,7 +128,7 @@ if family == socket.AF_UNSPEC: afs = [socket.AF_INET] - if hasattr(socket, 'AF_INET6'): + if _HAS_IPv6: afs.append(socket.AF_INET6) else: afs = [family] @@ -139,7 +144,10 @@ try: socket.inet_pton(af, host) # The host has already been resolved. - return af, type, proto, '', (host, port) + if _HAS_IPv6 and af == socket.AF_INET6: + return af, type, proto, '', (host, port, 0, 0) + else: + return af, type, proto, '', (host, port) except OSError: pass @@ -463,10 +471,7 @@ def _asyncgen_finalizer_hook(self, agen): self._asyncgens.discard(agen) if not self.is_closed(): - self.create_task(agen.aclose()) - # Wake up the loop if the finalizer was called from - # a different thread. - self._write_to_self() + self.call_soon_threadsafe(self.create_task, agen.aclose()) def _asyncgen_firstiter_hook(self, agen): if self._asyncgens_shutdown_called: @@ -1309,7 +1314,6 @@ raise ValueError( 'host/port and sock can not be specified at the same time') - AF_INET6 = getattr(socket, 'AF_INET6', 0) if reuse_address is None: reuse_address = os.name == 'posix' and sys.platform != 'cygwin' sockets = [] @@ -1349,7 +1353,9 @@ # Disable IPv4/IPv6 dual stack support (enabled by # default on Linux) which makes a single socket # listen on both address families. - if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'): + if (_HAS_IPv6 and + af == socket.AF_INET6 and + hasattr(socket, 'IPPROTO_IPV6')): sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True) @@ -1696,7 +1702,7 @@ elif self._scheduled: # Compute the desired timeout. when = self._scheduled[0]._when - timeout = max(0, when - self.time()) + timeout = min(max(0, when - self.time()), MAXIMUM_SELECT_TIMEOUT) if self._debug and timeout != 0: t0 = self.time() diff -Nru python3.7-3.7.0/Lib/asyncio/proactor_events.py python3.7-3.7.1/Lib/asyncio/proactor_events.py --- python3.7-3.7.0/Lib/asyncio/proactor_events.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/asyncio/proactor_events.py 2018-10-20 06:04:19.000000000 +0000 @@ -343,6 +343,10 @@ def _loop_writing(self, f=None, data=None): try: + if f is not None and self._write_fut is None and self._closing: + # XXX most likely self._force_close() has been called, and + # it has set self._write_fut to None. + return assert f is self._write_fut self._write_fut = None self._pending_write = 0 diff -Nru python3.7-3.7.0/Lib/asyncio/tasks.py python3.7-3.7.1/Lib/asyncio/tasks.py --- python3.7-3.7.0/Lib/asyncio/tasks.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/asyncio/tasks.py 2018-10-20 06:04:19.000000000 +0000 @@ -35,7 +35,9 @@ """Return a set of all tasks for the loop.""" if loop is None: loop = events.get_running_loop() - return {t for t in _all_tasks + # NB: set(_all_tasks) is required to protect + # from https://bugs.python.org/issue34970 bug + return {t for t in list(_all_tasks) if futures._get_loop(t) is loop and not t.done()} @@ -45,7 +47,9 @@ # method. if loop is None: loop = events.get_event_loop() - return {t for t in _all_tasks if futures._get_loop(t) is loop} + # NB: set(_all_tasks) is required to protect + # from https://bugs.python.org/issue34970 bug + return {t for t in list(_all_tasks) if futures._get_loop(t) is loop} class Task(futures._PyFuture): # Inherit Python Task implementation diff -Nru python3.7-3.7.0/Lib/base64.py python3.7-3.7.1/Lib/base64.py --- python3.7-3.7.0/Lib/base64.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/base64.py 2018-10-20 06:04:19.000000000 +0000 @@ -231,23 +231,16 @@ raise binascii.Error('Non-base32 digit found') from None decoded += acc.to_bytes(5, 'big') # Process the last, partial quanta - if padchars: + if l % 8 or padchars not in {0, 1, 3, 4, 6}: + raise binascii.Error('Incorrect padding') + if padchars and decoded: acc <<= 5 * padchars last = acc.to_bytes(5, 'big') - if padchars == 1: - decoded[-5:] = last[:-1] - elif padchars == 3: - decoded[-5:] = last[:-2] - elif padchars == 4: - decoded[-5:] = last[:-3] - elif padchars == 6: - decoded[-5:] = last[:-4] - else: - raise binascii.Error('Incorrect padding') + leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 + decoded[-5:] = last[:leftover] return bytes(decoded) - # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. diff -Nru python3.7-3.7.0/Lib/bdb.py python3.7-3.7.1/Lib/bdb.py --- python3.7-3.7.0/Lib/bdb.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/bdb.py 2018-10-20 06:04:19.000000000 +0000 @@ -74,7 +74,7 @@ return: A function or other code block is about to return. exception: An exception has occurred. c_call: A C function is about to be called. - c_return: A C functon has returned. + c_return: A C function has returned. c_exception: A C function has raised an exception. For the Python events, specialized functions (see the dispatch_*() diff -Nru python3.7-3.7.0/Lib/concurrent/futures/_base.py python3.7-3.7.1/Lib/concurrent/futures/_base.py --- python3.7-3.7.0/Lib/concurrent/futures/_base.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/concurrent/futures/_base.py 2018-10-20 06:04:19.000000000 +0000 @@ -212,7 +212,7 @@ before the given timeout. """ if timeout is not None: - end_time = timeout + time.time() + end_time = timeout + time.monotonic() fs = set(fs) total_futures = len(fs) @@ -231,7 +231,7 @@ if timeout is None: wait_timeout = None else: - wait_timeout = end_time - time.time() + wait_timeout = end_time - time.monotonic() if wait_timeout < 0: raise TimeoutError( '%d (of %d) futures unfinished' % ( @@ -570,7 +570,7 @@ Exception: If fn(*args) raises for any values. """ if timeout is not None: - end_time = timeout + time.time() + end_time = timeout + time.monotonic() fs = [self.submit(fn, *args) for args in zip(*iterables)] @@ -585,7 +585,7 @@ if timeout is None: yield fs.pop().result() else: - yield fs.pop().result(end_time - time.time()) + yield fs.pop().result(end_time - time.monotonic()) finally: for future in fs: future.cancel() diff -Nru python3.7-3.7.0/Lib/configparser.py python3.7-3.7.1/Lib/configparser.py --- python3.7-3.7.0/Lib/configparser.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/configparser.py 2018-10-20 06:04:19.000000000 +0000 @@ -80,7 +80,7 @@ Return list of configuration options for the named section. read(filenames, encoding=None) - Read and parse the list of named configuration files, given by + Read and parse the iterable of named configuration files, given by name. A single filename is also allowed. Non-existing files are ignored. Return list of successfully read files. @@ -676,13 +676,13 @@ return list(opts.keys()) def read(self, filenames, encoding=None): - """Read and parse a filename or a list of filenames. + """Read and parse a filename or an iterable of filenames. Files that cannot be opened are silently ignored; this is - designed so that you can specify a list of potential + designed so that you can specify an iterable of potential configuration file locations (e.g. current directory, user's home directory, systemwide directory), and all existing - configuration files in the list will be read. A single + configuration files in the iterable will be read. A single filename may also be given. Return list of successfully read files. diff -Nru python3.7-3.7.0/Lib/cProfile.py python3.7-3.7.1/Lib/cProfile.py --- python3.7-3.7.0/Lib/cProfile.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/cProfile.py 2018-10-20 06:04:19.000000000 +0000 @@ -25,11 +25,11 @@ # ____________________________________________________________ class Profile(_lsprof.Profiler): - """Profile(custom_timer=None, time_unit=None, subcalls=True, builtins=True) + """Profile(timer=None, timeunit=None, subcalls=True, builtins=True) Builds a profiler object using the specified timer function. The default timer is a fast built-in one based on real time. - For custom timer functions returning integers, time_unit can + For custom timer functions returning integers, timeunit can be a float specifying a scale (i.e. how long each integer unit is, in seconds). """ diff -Nru python3.7-3.7.0/Lib/ctypes/_aix.py python3.7-3.7.1/Lib/ctypes/_aix.py --- python3.7-3.7.0/Lib/ctypes/_aix.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/ctypes/_aix.py 2018-10-20 06:04:19.000000000 +0000 @@ -115,7 +115,7 @@ else: break p.stdout.close() - p.wait + p.wait() return ldr_headers def get_shared(ld_headers): diff -Nru python3.7-3.7.0/Lib/ctypes/test/test_as_parameter.py python3.7-3.7.1/Lib/ctypes/test/test_as_parameter.py --- python3.7-3.7.0/Lib/ctypes/test/test_as_parameter.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/ctypes/test/test_as_parameter.py 2018-10-20 06:04:19.000000000 +0000 @@ -24,7 +24,7 @@ f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] result = f(self.wrap(1), self.wrap("x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0)) self.assertEqual(result, 139) - self.assertTrue(type(result), int) + self.assertIs(type(result), int) def test_pointers(self): f = dll._testfunc_p_p diff -Nru python3.7-3.7.0/Lib/ctypes/test/test_win32.py python3.7-3.7.1/Lib/ctypes/test/test_win32.py --- python3.7-3.7.0/Lib/ctypes/test/test_win32.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/ctypes/test/test_win32.py 2018-10-20 06:04:19.000000000 +0000 @@ -55,6 +55,24 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') +class ReturnStructSizesTestCase(unittest.TestCase): + def test_sizes(self): + dll = CDLL(_ctypes_test.__file__) + for i in range(1, 11): + fields = [ (f"f{f}", c_char) for f in range(1, i + 1)] + class S(Structure): + _fields_ = fields + f = getattr(dll, f"TestSize{i}") + f.restype = S + res = f() + for i, f in enumerate(fields): + value = getattr(res, f[0]) + expected = bytes([ord('a') + i]) + self.assertEqual(value, expected) + + + +@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWintypes(unittest.TestCase): def test_HWND(self): from ctypes import wintypes diff -Nru python3.7-3.7.0/Lib/dataclasses.py python3.7-3.7.1/Lib/dataclasses.py --- python3.7-3.7.0/Lib/dataclasses.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/dataclasses.py 2018-10-20 06:04:19.000000000 +0000 @@ -4,6 +4,7 @@ import types import inspect import keyword +import builtins __all__ = ['dataclass', 'field', @@ -257,7 +258,7 @@ # This is used to support the PEP 487 __set_name__ protocol in the # case where we're using a field that contains a descriptor as a - # defaul value. For details on __set_name__, see + # default value. For details on __set_name__, see # https://www.python.org/dev/peps/pep-0487/#implementation-details. # # Note that in _process_class, this Field object is overwritten @@ -343,6 +344,11 @@ # worries about external callers. if locals is None: locals = {} + # __builtins__ may be the "builtins" module or + # the value of its "__dict__", + # so make sure "__builtins__" is the module. + if globals is not None and '__builtins__' not in globals: + globals['__builtins__'] = builtins return_annotation = '' if return_type is not MISSING: locals['_return_type'] = return_type @@ -365,7 +371,7 @@ # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'object.__setattr__({self_name},{name!r},{value})' + return f'__builtins__.object.__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' @@ -1020,11 +1026,36 @@ value = _asdict_inner(getattr(obj, f.name), dict_factory) result.append((f.name, value)) return dict_factory(result) + elif isinstance(obj, tuple) and hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + + # I'm not using namedtuple's _asdict() + # method, because: + # - it does not recurse in to the namedtuple fields and + # convert them to dicts (using dict_factory). + # - I don't actually want to return a dict here. The the main + # use case here is json.dumps, and it handles converting + # namedtuples to lists. Admittedly we're losing some + # information here when we produce a json list instead of a + # dict. Note that if we returned dicts here instead of + # namedtuples, we could no longer call asdict() on a data + # structure where a namedtuple was used as a dict key. + + return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). return type(obj)(_asdict_inner(v, dict_factory) for v in obj) elif isinstance(obj, dict): - return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) - for k, v in obj.items()) + return type(obj)((_asdict_inner(k, dict_factory), + _asdict_inner(v, dict_factory)) + for k, v in obj.items()) else: return copy.deepcopy(obj) @@ -1060,7 +1091,18 @@ value = _astuple_inner(getattr(obj, f.name), tuple_factory) result.append(value) return tuple_factory(result) + elif isinstance(obj, tuple) and hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + return type(obj)(*[_astuple_inner(v, tuple_factory) for v in obj]) elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) elif isinstance(obj, dict): return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) @@ -1173,6 +1215,9 @@ continue if f.name not in changes: + if f._field_type is _FIELD_INITVAR: + raise ValueError(f"InitVar {f.name!r} " + 'must be specified with replace()') changes[f.name] = getattr(obj, f.name) # Create the new object, which calls __init__() and diff -Nru python3.7-3.7.0/Lib/datetime.py python3.7-3.7.1/Lib/datetime.py --- python3.7-3.7.0/Lib/datetime.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/datetime.py 2018-10-20 06:04:19.000000000 +0000 @@ -6,6 +6,7 @@ import time as _time import math as _math +import sys def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 @@ -1572,6 +1573,14 @@ # 23 hours at 1969-09-30 13:00:00 in Kwajalein. # Let's probe 24 hours in the past to detect a transition: max_fold_seconds = 24 * 3600 + + # On Windows localtime_s throws an OSError for negative values, + # thus we can't perform fold detection for values of time less + # than the max time fold. See comments in _datetimemodule's + # version of this method for more details. + if t < max_fold_seconds and sys.platform.startswith("win"): + return result + y, m, d, hh, mm, ss = converter(t - max_fold_seconds)[:6] probe1 = cls(y, m, d, hh, mm, ss, us, tz) trans = result - probe1 - timedelta(0, max_fold_seconds) diff -Nru python3.7-3.7.0/Lib/distutils/log.py python3.7-3.7.1/Lib/distutils/log.py --- python3.7-3.7.0/Lib/distutils/log.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/distutils/log.py 2018-10-20 06:04:19.000000000 +0000 @@ -27,11 +27,13 @@ stream = sys.stderr else: stream = sys.stdout - if stream.errors == 'strict': + try: + stream.write('%s\n' % msg) + except UnicodeEncodeError: # emulate backslashreplace error handler encoding = stream.encoding msg = msg.encode(encoding, "backslashreplace").decode(encoding) - stream.write('%s\n' % msg) + stream.write('%s\n' % msg) stream.flush() def log(self, level, msg, *args): diff -Nru python3.7-3.7.0/Lib/distutils/_msvccompiler.py python3.7-3.7.1/Lib/distutils/_msvccompiler.py --- python3.7-3.7.0/Lib/distutils/_msvccompiler.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/distutils/_msvccompiler.py 2018-10-20 06:04:19.000000000 +0000 @@ -252,11 +252,11 @@ for dir in vc_env.get('include', '').split(os.pathsep): if dir: - self.add_include_dir(dir) + self.add_include_dir(dir.rstrip(os.sep)) for dir in vc_env.get('lib', '').split(os.pathsep): if dir: - self.add_library_dir(dir) + self.add_library_dir(dir.rstrip(os.sep)) self.preprocess_options = None # If vcruntime_redist is available, link against it dynamically. Otherwise, diff -Nru python3.7-3.7.0/Lib/distutils/spawn.py python3.7-3.7.1/Lib/distutils/spawn.py --- python3.7-3.7.0/Lib/distutils/spawn.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/distutils/spawn.py 2018-10-20 06:04:19.000000000 +0000 @@ -173,7 +173,7 @@ os.environ['PATH']. Returns the complete filename or None if not found. """ if path is None: - path = os.environ['PATH'] + path = os.environ.get('PATH', os.defpath) paths = path.split(os.pathsep) base, ext = os.path.splitext(executable) diff -Nru python3.7-3.7.0/Lib/distutils/tests/test_bdist.py python3.7-3.7.1/Lib/distutils/tests/test_bdist.py --- python3.7-3.7.0/Lib/distutils/tests/test_bdist.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/distutils/tests/test_bdist.py 2018-10-20 06:04:19.000000000 +0000 @@ -39,6 +39,9 @@ for name in names: subcmd = cmd.get_finalized_command(name) + if getattr(subcmd, '_unsupported', False): + # command is not supported on this build + continue self.assertTrue(subcmd.skip_build, '%s should take --skip-build from bdist' % name) diff -Nru python3.7-3.7.0/Lib/distutils/tests/test_bdist_wininst.py python3.7-3.7.1/Lib/distutils/tests/test_bdist_wininst.py --- python3.7-3.7.0/Lib/distutils/tests/test_bdist_wininst.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/distutils/tests/test_bdist_wininst.py 2018-10-20 06:04:19.000000000 +0000 @@ -5,6 +5,8 @@ from distutils.command.bdist_wininst import bdist_wininst from distutils.tests import support +@unittest.skipIf(getattr(bdist_wininst, '_unsupported', False), + 'bdist_wininst is not supported in this install') class BuildWinInstTestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): diff -Nru python3.7-3.7.0/Lib/distutils/tests/test_log.py python3.7-3.7.1/Lib/distutils/tests/test_log.py --- python3.7-3.7.0/Lib/distutils/tests/test_log.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/distutils/tests/test_log.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,35 +1,43 @@ """Tests for distutils.log""" +import io import sys import unittest -from tempfile import NamedTemporaryFile -from test.support import run_unittest +from test.support import swap_attr, run_unittest from distutils import log class TestLog(unittest.TestCase): def test_non_ascii(self): - # Issue #8663: test that non-ASCII text is escaped with - # backslashreplace error handler (stream use ASCII encoding and strict - # error handler) - old_stdout = sys.stdout - old_stderr = sys.stderr - old_threshold = log.set_threshold(log.DEBUG) - try: - with NamedTemporaryFile(mode="w+", encoding='ascii') as stdout, \ - NamedTemporaryFile(mode="w+", encoding='ascii') as stderr: - sys.stdout = stdout - sys.stderr = stderr - log.debug("debug:\xe9") - log.fatal("fatal:\xe9") + # Issues #8663, #34421: test that non-encodable text is escaped with + # backslashreplace error handler and encodable non-ASCII text is + # output as is. + for errors in ('strict', 'backslashreplace', 'surrogateescape', + 'replace', 'ignore'): + with self.subTest(errors=errors): + stdout = io.TextIOWrapper(io.BytesIO(), + encoding='cp437', errors=errors) + stderr = io.TextIOWrapper(io.BytesIO(), + encoding='cp437', errors=errors) + old_threshold = log.set_threshold(log.DEBUG) + try: + with swap_attr(sys, 'stdout', stdout), \ + swap_attr(sys, 'stderr', stderr): + log.debug('Dεbug\tMėssãge') + log.fatal('Fαtal\tÈrrōr') + finally: + log.set_threshold(old_threshold) + stdout.seek(0) - self.assertEqual(stdout.read().rstrip(), "debug:\\xe9") + self.assertEqual(stdout.read().rstrip(), + 'Dεbug\tM?ss?ge' if errors == 'replace' else + 'Dεbug\tMssge' if errors == 'ignore' else + 'Dεbug\tM\\u0117ss\\xe3ge') stderr.seek(0) - self.assertEqual(stderr.read().rstrip(), "fatal:\\xe9") - finally: - log.set_threshold(old_threshold) - sys.stdout = old_stdout - sys.stderr = old_stderr + self.assertEqual(stderr.read().rstrip(), + 'Fαtal\t?rr?r' if errors == 'replace' else + 'Fαtal\trrr' if errors == 'ignore' else + 'Fαtal\t\\xc8rr\\u014dr') def test_suite(): return unittest.makeSuite(TestLog) diff -Nru python3.7-3.7.0/Lib/distutils/tests/test_spawn.py python3.7-3.7.1/Lib/distutils/tests/test_spawn.py --- python3.7-3.7.0/Lib/distutils/tests/test_spawn.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/distutils/tests/test_spawn.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,9 +1,13 @@ """Tests for distutils.spawn.""" -import unittest -import sys import os +import stat +import sys +import unittest +from unittest import mock from test.support import run_unittest, unix_shell +from test import support as test_support +from distutils.spawn import find_executable from distutils.spawn import _nt_quote_args from distutils.spawn import spawn from distutils.errors import DistutilsExecError @@ -51,6 +55,47 @@ os.chmod(exe, 0o777) spawn([exe]) # should work without any error + def test_find_executable(self): + with test_support.temp_dir() as tmp_dir: + # use TESTFN to get a pseudo-unique filename + program_noeext = test_support.TESTFN + # Give the temporary program an ".exe" suffix for all. + # It's needed on Windows and not harmful on other platforms. + program = program_noeext + ".exe" + + filename = os.path.join(tmp_dir, program) + with open(filename, "wb"): + pass + os.chmod(filename, stat.S_IXUSR) + + # test path parameter + rv = find_executable(program, path=tmp_dir) + self.assertEqual(rv, filename) + + if sys.platform == 'win32': + # test without ".exe" extension + rv = find_executable(program_noeext, path=tmp_dir) + self.assertEqual(rv, filename) + + # test find in the current directory + with test_support.change_cwd(tmp_dir): + rv = find_executable(program) + self.assertEqual(rv, program) + + # test non-existent program + dont_exist_program = "dontexist_" + program + rv = find_executable(dont_exist_program , path=tmp_dir) + self.assertIsNone(rv) + + # test os.defpath: missing PATH environment variable + with test_support.EnvironmentVarGuard() as env: + with mock.patch('distutils.spawn.os.defpath', tmp_dir): + env.pop('PATH') + + rv = find_executable(program) + self.assertEqual(rv, filename) + + def test_suite(): return unittest.makeSuite(SpawnTestCase) diff -Nru python3.7-3.7.0/Lib/email/_encoded_words.py python3.7-3.7.1/Lib/email/_encoded_words.py --- python3.7-3.7.0/Lib/email/_encoded_words.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/email/_encoded_words.py 2018-10-20 06:04:19.000000000 +0000 @@ -98,30 +98,42 @@ # def decode_b(encoded): - defects = [] + # First try encoding with validate=True, fixing the padding if needed. + # This will succeed only if encoded includes no invalid characters. pad_err = len(encoded) % 4 - if pad_err: - defects.append(errors.InvalidBase64PaddingDefect()) - padded_encoded = encoded + b'==='[:4-pad_err] - else: - padded_encoded = encoded + missing_padding = b'==='[:4-pad_err] if pad_err else b'' try: - return base64.b64decode(padded_encoded, validate=True), defects + return ( + base64.b64decode(encoded + missing_padding, validate=True), + [errors.InvalidBase64PaddingDefect()] if pad_err else [], + ) except binascii.Error: - # Since we had correct padding, this must an invalid char error. - defects = [errors.InvalidBase64CharactersDefect()] + # Since we had correct padding, this is likely an invalid char error. + # # The non-alphabet characters are ignored as far as padding - # goes, but we don't know how many there are. So we'll just - # try various padding lengths until something works. - for i in 0, 1, 2, 3: + # goes, but we don't know how many there are. So try without adding + # padding to see if it works. + try: + return ( + base64.b64decode(encoded, validate=False), + [errors.InvalidBase64CharactersDefect()], + ) + except binascii.Error: + # Add as much padding as could possibly be necessary (extra padding + # is ignored). try: - return base64.b64decode(encoded+b'='*i, validate=False), defects + return ( + base64.b64decode(encoded + b'==', validate=False), + [errors.InvalidBase64CharactersDefect(), + errors.InvalidBase64PaddingDefect()], + ) except binascii.Error: - if i==0: - defects.append(errors.InvalidBase64PaddingDefect()) - else: - # This should never happen. - raise AssertionError("unexpected binascii.Error") + # This only happens when the encoded string's length is 1 more + # than a multiple of 4, which is invalid. + # + # bpo-27397: Just return the encoded string since there's no + # way to decode. + return encoded, [errors.InvalidBase64LengthDefect()] def encode_b(bstring): return base64.b64encode(bstring).decode('ascii') diff -Nru python3.7-3.7.0/Lib/email/errors.py python3.7-3.7.1/Lib/email/errors.py --- python3.7-3.7.0/Lib/email/errors.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/email/errors.py 2018-10-20 06:04:19.000000000 +0000 @@ -73,6 +73,9 @@ class InvalidBase64CharactersDefect(MessageDefect): """base64 encoded sequence had characters not in base64 alphabet""" +class InvalidBase64LengthDefect(MessageDefect): + """base64 encoded sequence had invalid length (1 mod 4)""" + # These errors are specific to header parsing. class HeaderDefect(MessageDefect): diff -Nru python3.7-3.7.0/Lib/email/_header_value_parser.py python3.7-3.7.1/Lib/email/_header_value_parser.py --- python3.7-3.7.0/Lib/email/_header_value_parser.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/email/_header_value_parser.py 2018-10-20 06:04:19.000000000 +0000 @@ -1875,7 +1875,7 @@ if not value: group.defects.append(errors.InvalidHeaderDefect( "end of header in group")) - if value[0] != ';': + elif value[0] != ';': raise errors.HeaderParseError( "expected ';' at end of group but found {}".format(value)) group.append(ValueTerminal(';', 'group-terminator')) diff -Nru python3.7-3.7.0/Lib/enum.py python3.7-3.7.1/Lib/enum.py --- python3.7-3.7.0/Lib/enum.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/enum.py 2018-10-20 06:04:19.000000000 +0000 @@ -171,9 +171,11 @@ enum_class._member_map_ = OrderedDict() # name->value map enum_class._member_type_ = member_type - # save attributes from super classes so we know if we can take - # the shortcut of storing members in the class dict - base_attributes = {a for b in enum_class.mro() for a in b.__dict__} + # save DynamicClassAttribute attributes from super classes so we know + # if we can take the shortcut of storing members in the class dict + dynamic_attributes = {k for c in enum_class.mro() + for k, v in c.__dict__.items() + if isinstance(v, DynamicClassAttribute)} # Reverse value->name map for hashable values. enum_class._value2member_map_ = {} @@ -233,7 +235,7 @@ enum_class._member_names_.append(member_name) # performance boost for any member that would not shadow # a DynamicClassAttribute - if member_name not in base_attributes: + if member_name not in dynamic_attributes: setattr(enum_class, member_name, enum_member) # now add to _member_map_ enum_class._member_map_[member_name] = enum_member @@ -447,38 +449,25 @@ if not bases: return object, Enum - # double check that we are not subclassing a class with existing - # enumeration members; while we're at it, see if any other data - # type has been mixed in so we can use the correct __new__ - member_type = first_enum = None - for base in bases: - if (base is not Enum and - issubclass(base, Enum) and - base._member_names_): - raise TypeError("Cannot extend enumerations") - # base is now the last base in bases - if not issubclass(base, Enum): - raise TypeError("new enumerations must be created as " - "`ClassName([mixin_type,] enum_type)`") - - # get correct mix-in type (either mix-in type of Enum subclass, or - # first base if last base is Enum) - if not issubclass(bases[0], Enum): - member_type = bases[0] # first data type - first_enum = bases[-1] # enum type - else: - for base in bases[0].__mro__: - # most common: (IntEnum, int, Enum, object) - # possible: (, , - # , , - # ) - if issubclass(base, Enum): - if first_enum is None: - first_enum = base - else: - if member_type is None: - member_type = base - + def _find_data_type(bases): + for chain in bases: + for base in chain.__mro__: + if base is object: + continue + elif '__new__' in base.__dict__: + if issubclass(base, Enum): + continue + return base + + # ensure final parent class is an Enum derivative, find any concrete + # data type, and check that Enum has no members + first_enum = bases[-1] + if not issubclass(first_enum, Enum): + raise TypeError("new enumerations should be created as " + "`EnumName([mixin_type, ...] [data_type,] enum_type)`") + member_type = _find_data_type(bases) or object + if first_enum._member_names_: + raise TypeError("Cannot extend enumerations") return member_type, first_enum @staticmethod @@ -524,7 +513,6 @@ use_args = False else: use_args = True - return __new__, save_new, use_args diff -Nru python3.7-3.7.0/Lib/functools.py python3.7-3.7.1/Lib/functools.py --- python3.7-3.7.0/Lib/functools.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/functools.py 2018-10-20 06:04:19.000000000 +0000 @@ -817,8 +817,13 @@ return func def wrapper(*args, **kw): + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') + return dispatch(args[0].__class__)(*args, **kw) + funcname = getattr(func, '__name__', 'singledispatch function') registry[object] = func wrapper.register = register wrapper.dispatch = dispatch diff -Nru python3.7-3.7.0/Lib/hashlib.py python3.7-3.7.1/Lib/hashlib.py --- python3.7-3.7.0/Lib/hashlib.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/hashlib.py 2018-10-20 06:04:19.000000000 +0000 @@ -25,18 +25,18 @@ sha384 and sha512 will be slow on 32 bit platforms. Hash objects have these methods: - - update(arg): Update the hash object with the bytes in arg. Repeated calls - are equivalent to a single call with the concatenation of all - the arguments. - - digest(): Return the digest of the bytes passed to the update() method - so far. - - hexdigest(): Like digest() except the digest is returned as a unicode - object of double length, containing only hexadecimal digits. - - copy(): Return a copy (clone) of the hash object. This can be used to - efficiently compute the digests of strings that share a common - initial substring. + - update(data): Update the hash object with the bytes in data. Repeated calls + are equivalent to a single call with the concatenation of all + the arguments. + - digest(): Return the digest of the bytes passed to the update() method + so far as a bytes object. + - hexdigest(): Like digest() except the digest is returned as a string + of double length, containing only hexadecimal digits. + - copy(): Return a copy (clone) of the hash object. This can be used to + efficiently compute the digests of datas that share a common + initial substring. -For example, to obtain the digest of the string 'Nobody inspects the +For example, to obtain the digest of the byte string 'Nobody inspects the spammish repetition': >>> import hashlib @@ -130,14 +130,15 @@ def __py_new(name, data=b'', **kwargs): """new(name, data=b'', **kwargs) - Return a new hashing object using the - named algorithm; optionally initialized with data (which must be bytes). + named algorithm; optionally initialized with data (which must be + a bytes-like object). """ return __get_builtin_constructor(name)(data, **kwargs) def __hash_new(name, data=b'', **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; - optionally initialized with data (which must be bytes). + optionally initialized with data (which must be a bytes-like object). """ if name in {'blake2b', 'blake2s'}: # Prefer our blake2 implementation. diff -Nru python3.7-3.7.0/Lib/http/client.py python3.7-3.7.1/Lib/http/client.py --- python3.7-3.7.0/Lib/http/client.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/http/client.py 2018-10-20 06:04:19.000000000 +0000 @@ -321,7 +321,7 @@ if self.debuglevel > 0: for hdr in self.headers: - print("header:", hdr, end=" ") + print("header:", hdr + ":", self.headers.get(hdr)) # are we using the chunked-style of transfer encoding? tr_enc = self.headers.get("transfer-encoding") diff -Nru python3.7-3.7.0/Lib/http/server.py python3.7-3.7.1/Lib/http/server.py --- python3.7-3.7.0/Lib/http/server.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/http/server.py 2018-10-20 06:04:19.000000000 +0000 @@ -474,7 +474,7 @@ }) body = content.encode('UTF-8', 'replace') self.send_header("Content-Type", self.error_content_type) - self.send_header('Content-Length', int(len(body))) + self.send_header('Content-Length', str(len(body))) self.end_headers() if self.command != 'HEAD' and body: diff -Nru python3.7-3.7.0/Lib/idlelib/autocomplete.py python3.7-3.7.1/Lib/idlelib/autocomplete.py --- python3.7-3.7.0/Lib/idlelib/autocomplete.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/autocomplete.py 2018-10-20 06:04:19.000000000 +0000 @@ -226,7 +226,6 @@ AutoComplete.reload() - if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_autocomplete', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/autocomplete_w.py python3.7-3.7.1/Lib/idlelib/autocomplete_w.py --- python3.7-3.7.0/Lib/idlelib/autocomplete_w.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/autocomplete_w.py 2018-10-20 06:04:19.000000000 +0000 @@ -246,7 +246,7 @@ acw.wm_geometry("+%d+%d" % (new_x, new_y)) if platform.system().startswith('Windows'): - # See issue 15786. When on windows platform, Tk will misbehave + # See issue 15786. When on Windows platform, Tk will misbehave # to call winconfig_event multiple times, we need to prevent this, # otherwise mouse button double click will not be able to used. acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid) @@ -269,7 +269,7 @@ # mouse click on widget / text area. if self.is_active(): if event.type == EventType.FocusOut: - # On windows platform, it will need to delay the check for + # On Windows platform, it will need to delay the check for # acw.focus_get() when click on acw, otherwise it will return # None and close the window self.widget.after(1, self._hide_event_check) @@ -458,3 +458,10 @@ self.listbox = None self.autocompletewindow.destroy() self.autocompletewindow = None + + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_autocomplete_w', verbosity=2, exit=False) + +# TODO: autocomplete/w htest here diff -Nru python3.7-3.7.0/Lib/idlelib/autoexpand.py python3.7-3.7.1/Lib/idlelib/autoexpand.py --- python3.7-3.7.0/Lib/idlelib/autoexpand.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/autoexpand.py 2018-10-20 06:04:19.000000000 +0000 @@ -92,5 +92,5 @@ if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_autoexpand', verbosity=2) + from unittest import main + main('idlelib.idle_test.test_autoexpand', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/browser.py python3.7-3.7.1/Lib/idlelib/browser.py --- python3.7-3.7.0/Lib/idlelib/browser.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/browser.py 2018-10-20 06:04:19.000000000 +0000 @@ -16,7 +16,7 @@ from idlelib.config import idleConf from idlelib import pyshell from idlelib.tree import TreeNode, TreeItem, ScrolledCanvas -from idlelib.windows import ListedToplevel +from idlelib.window import ListedToplevel file_open = None # Method...Item and Class...Item use this. diff -Nru python3.7-3.7.0/Lib/idlelib/calltip.py python3.7-3.7.1/Lib/idlelib/calltip.py --- python3.7-3.7.0/Lib/idlelib/calltip.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/calltip.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,178 @@ +"""Pop up a reminder of how to call a function. + +Call Tips are floating windows which display function, class, and method +parameter and docstring information when you type an opening parenthesis, and +which disappear when you type a closing parenthesis. +""" +import inspect +import re +import sys +import textwrap +import types + +from idlelib import calltip_w +from idlelib.hyperparser import HyperParser +import __main__ + + +class Calltip: + + def __init__(self, editwin=None): + if editwin is None: # subprocess and test + self.editwin = None + else: + self.editwin = editwin + self.text = editwin.text + self.active_calltip = None + self._calltip_window = self._make_tk_calltip_window + + def close(self): + self._calltip_window = None + + def _make_tk_calltip_window(self): + # See __init__ for usage + return calltip_w.CalltipWindow(self.text) + + def _remove_calltip_window(self, event=None): + if self.active_calltip: + self.active_calltip.hidetip() + self.active_calltip = None + + def force_open_calltip_event(self, event): + "The user selected the menu entry or hotkey, open the tip." + self.open_calltip(True) + return "break" + + def try_open_calltip_event(self, event): + """Happens when it would be nice to open a calltip, but not really + necessary, for example after an opening bracket, so function calls + won't be made. + """ + self.open_calltip(False) + + def refresh_calltip_event(self, event): + if self.active_calltip and self.active_calltip.tipwindow: + self.open_calltip(False) + + def open_calltip(self, evalfuncs): + self._remove_calltip_window() + + hp = HyperParser(self.editwin, "insert") + sur_paren = hp.get_surrounding_brackets('(') + if not sur_paren: + return + hp.set_index(sur_paren[0]) + expression = hp.get_expression() + if not expression: + return + if not evalfuncs and (expression.find('(') != -1): + return + argspec = self.fetch_tip(expression) + if not argspec: + return + self.active_calltip = self._calltip_window() + self.active_calltip.showtip(argspec, sur_paren[0], sur_paren[1]) + + def fetch_tip(self, expression): + """Return the argument list and docstring of a function or class. + + If there is a Python subprocess, get the calltip there. Otherwise, + either this fetch_tip() is running in the subprocess or it was + called in an IDLE running without the subprocess. + + The subprocess environment is that of the most recently run script. If + two unrelated modules are being edited some calltips in the current + module may be inoperative if the module was not the last to run. + + To find methods, fetch_tip must be fed a fully qualified name. + + """ + try: + rpcclt = self.editwin.flist.pyshell.interp.rpcclt + except AttributeError: + rpcclt = None + if rpcclt: + return rpcclt.remotecall("exec", "get_the_calltip", + (expression,), {}) + else: + return get_argspec(get_entity(expression)) + + +def get_entity(expression): + """Return the object corresponding to expression evaluated + in a namespace spanning sys.modules and __main.dict__. + """ + if expression: + namespace = sys.modules.copy() + namespace.update(__main__.__dict__) + try: + return eval(expression, namespace) + except BaseException: + # An uncaught exception closes idle, and eval can raise any + # exception, especially if user classes are involved. + return None + +# The following are used in get_argspec and some in tests +_MAX_COLS = 85 +_MAX_LINES = 5 # enough for bytes +_INDENT = ' '*4 # for wrapped signatures +_first_param = re.compile(r'(?<=\()\w*\,?\s*') +_default_callable_argspec = "See source or doc" +_invalid_method = "invalid method signature" +_argument_positional = "\n['/' marks preceding arguments as positional-only]\n" + +def get_argspec(ob): + '''Return a string describing the signature of a callable object, or ''. + + For Python-coded functions and methods, the first line is introspected. + Delete 'self' parameter for classes (.__init__) and bound methods. + The next lines are the first lines of the doc string up to the first + empty line or _MAX_LINES. For builtins, this typically includes + the arguments in addition to the return value. + ''' + argspec = default = "" + try: + ob_call = ob.__call__ + except BaseException: + return default + + fob = ob_call if isinstance(ob_call, types.MethodType) else ob + + try: + argspec = str(inspect.signature(fob)) + except ValueError as err: + msg = str(err) + if msg.startswith(_invalid_method): + return _invalid_method + + if '/' in argspec: + """Using AC's positional argument should add the explain""" + argspec += _argument_positional + if isinstance(fob, type) and argspec == '()': + """fob with no argument, use default callable argspec""" + argspec = _default_callable_argspec + + lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) + if len(argspec) > _MAX_COLS else [argspec] if argspec else []) + + if isinstance(ob_call, types.MethodType): + doc = ob_call.__doc__ + else: + doc = getattr(ob, "__doc__", "") + if doc: + for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: + line = line.strip() + if not line: + break + if len(line) > _MAX_COLS: + line = line[: _MAX_COLS - 3] + '...' + lines.append(line) + argspec = '\n'.join(lines) + if not argspec: + argspec = _default_callable_argspec + return argspec + + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_calltip', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/calltips.py python3.7-3.7.1/Lib/idlelib/calltips.py --- python3.7-3.7.0/Lib/idlelib/calltips.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/calltips.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,178 +0,0 @@ -"""Pop up a reminder of how to call a function. - -Call Tips are floating windows which display function, class, and method -parameter and docstring information when you type an opening parenthesis, and -which disappear when you type a closing parenthesis. -""" -import inspect -import re -import sys -import textwrap -import types - -from idlelib import calltip_w -from idlelib.hyperparser import HyperParser -import __main__ - - -class CallTips: - - def __init__(self, editwin=None): - if editwin is None: # subprocess and test - self.editwin = None - else: - self.editwin = editwin - self.text = editwin.text - self.active_calltip = None - self._calltip_window = self._make_tk_calltip_window - - def close(self): - self._calltip_window = None - - def _make_tk_calltip_window(self): - # See __init__ for usage - return calltip_w.CallTip(self.text) - - def _remove_calltip_window(self, event=None): - if self.active_calltip: - self.active_calltip.hidetip() - self.active_calltip = None - - def force_open_calltip_event(self, event): - "The user selected the menu entry or hotkey, open the tip." - self.open_calltip(True) - return "break" - - def try_open_calltip_event(self, event): - """Happens when it would be nice to open a CallTip, but not really - necessary, for example after an opening bracket, so function calls - won't be made. - """ - self.open_calltip(False) - - def refresh_calltip_event(self, event): - if self.active_calltip and self.active_calltip.is_active(): - self.open_calltip(False) - - def open_calltip(self, evalfuncs): - self._remove_calltip_window() - - hp = HyperParser(self.editwin, "insert") - sur_paren = hp.get_surrounding_brackets('(') - if not sur_paren: - return - hp.set_index(sur_paren[0]) - expression = hp.get_expression() - if not expression: - return - if not evalfuncs and (expression.find('(') != -1): - return - argspec = self.fetch_tip(expression) - if not argspec: - return - self.active_calltip = self._calltip_window() - self.active_calltip.showtip(argspec, sur_paren[0], sur_paren[1]) - - def fetch_tip(self, expression): - """Return the argument list and docstring of a function or class. - - If there is a Python subprocess, get the calltip there. Otherwise, - either this fetch_tip() is running in the subprocess or it was - called in an IDLE running without the subprocess. - - The subprocess environment is that of the most recently run script. If - two unrelated modules are being edited some calltips in the current - module may be inoperative if the module was not the last to run. - - To find methods, fetch_tip must be fed a fully qualified name. - - """ - try: - rpcclt = self.editwin.flist.pyshell.interp.rpcclt - except AttributeError: - rpcclt = None - if rpcclt: - return rpcclt.remotecall("exec", "get_the_calltip", - (expression,), {}) - else: - return get_argspec(get_entity(expression)) - - -def get_entity(expression): - """Return the object corresponding to expression evaluated - in a namespace spanning sys.modules and __main.dict__. - """ - if expression: - namespace = sys.modules.copy() - namespace.update(__main__.__dict__) - try: - return eval(expression, namespace) - except BaseException: - # An uncaught exception closes idle, and eval can raise any - # exception, especially if user classes are involved. - return None - -# The following are used in get_argspec and some in tests -_MAX_COLS = 85 -_MAX_LINES = 5 # enough for bytes -_INDENT = ' '*4 # for wrapped signatures -_first_param = re.compile(r'(?<=\()\w*\,?\s*') -_default_callable_argspec = "See source or doc" -_invalid_method = "invalid method signature" -_argument_positional = "\n['/' marks preceding arguments as positional-only]\n" - -def get_argspec(ob): - '''Return a string describing the signature of a callable object, or ''. - - For Python-coded functions and methods, the first line is introspected. - Delete 'self' parameter for classes (.__init__) and bound methods. - The next lines are the first lines of the doc string up to the first - empty line or _MAX_LINES. For builtins, this typically includes - the arguments in addition to the return value. - ''' - argspec = default = "" - try: - ob_call = ob.__call__ - except BaseException: - return default - - fob = ob_call if isinstance(ob_call, types.MethodType) else ob - - try: - argspec = str(inspect.signature(fob)) - except ValueError as err: - msg = str(err) - if msg.startswith(_invalid_method): - return _invalid_method - - if '/' in argspec: - """Using AC's positional argument should add the explain""" - argspec += _argument_positional - if isinstance(fob, type) and argspec == '()': - """fob with no argument, use default callable argspec""" - argspec = _default_callable_argspec - - lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) - if len(argspec) > _MAX_COLS else [argspec] if argspec else []) - - if isinstance(ob_call, types.MethodType): - doc = ob_call.__doc__ - else: - doc = getattr(ob, "__doc__", "") - if doc: - for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: - line = line.strip() - if not line: - break - if len(line) > _MAX_COLS: - line = line[: _MAX_COLS - 3] + '...' - lines.append(line) - argspec = '\n'.join(lines) - if not argspec: - argspec = _default_callable_argspec - return argspec - - -if __name__ == '__main__': - from unittest import main - main('idlelib.idle_test.test_calltips', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/calltip_w.py python3.7-3.7.1/Lib/idlelib/calltip_w.py --- python3.7-3.7.0/Lib/idlelib/calltip_w.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/calltip_w.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,110 +1,118 @@ -"""A CallTip window class for Tkinter/IDLE. +"""A call-tip window class for Tkinter/IDLE. -After tooltip.py, which uses ideas gleaned from PySol -Used by the calltips IDLE extension. +After tooltip.py, which uses ideas gleaned from PySol. +Used by calltip.py. """ -from tkinter import Toplevel, Label, LEFT, SOLID, TclError +from tkinter import Label, LEFT, SOLID, TclError -HIDE_VIRTUAL_EVENT_NAME = "<>" +from idlelib.tooltip import TooltipBase + +HIDE_EVENT = "<>" HIDE_SEQUENCES = ("", "") -CHECKHIDE_VIRTUAL_EVENT_NAME = "<>" +CHECKHIDE_EVENT = "<>" CHECKHIDE_SEQUENCES = ("", "") -CHECKHIDE_TIME = 100 # milliseconds +CHECKHIDE_TIME = 100 # milliseconds MARK_RIGHT = "calltipwindowregion_right" -class CallTip: - def __init__(self, widget): - self.widget = widget - self.tipwindow = self.label = None - self.parenline = self.parencol = None - self.lastline = None +class CalltipWindow(TooltipBase): + """A call-tip widget for tkinter text widgets.""" + + def __init__(self, text_widget): + """Create a call-tip; shown by showtip(). + + text_widget: a Text widget with code for which call-tips are desired + """ + # Note: The Text widget will be accessible as self.anchor_widget + super(CalltipWindow, self).__init__(text_widget) + + self.label = self.text = None + self.parenline = self.parencol = self.lastline = None self.hideid = self.checkhideid = None self.checkhide_after_id = None - def position_window(self): - """Check if needs to reposition the window, and if so - do it.""" - curline = int(self.widget.index("insert").split('.')[0]) - if curline == self.lastline: - return - self.lastline = curline - self.widget.see("insert") + def get_position(self): + """Choose the position of the call-tip.""" + curline = int(self.anchor_widget.index("insert").split('.')[0]) if curline == self.parenline: - box = self.widget.bbox("%d.%d" % (self.parenline, - self.parencol)) + anchor_index = (self.parenline, self.parencol) else: - box = self.widget.bbox("%d.0" % curline) + anchor_index = (curline, 0) + box = self.anchor_widget.bbox("%d.%d" % anchor_index) if not box: - box = list(self.widget.bbox("insert")) + box = list(self.anchor_widget.bbox("insert")) # align to left of window box[0] = 0 box[2] = 0 - x = box[0] + self.widget.winfo_rootx() + 2 - y = box[1] + box[3] + self.widget.winfo_rooty() - self.tipwindow.wm_geometry("+%d+%d" % (x, y)) + return box[0] + 2, box[1] + box[3] + + def position_window(self): + "Reposition the window if needed." + curline = int(self.anchor_widget.index("insert").split('.')[0]) + if curline == self.lastline: + return + self.lastline = curline + self.anchor_widget.see("insert") + super(CalltipWindow, self).position_window() def showtip(self, text, parenleft, parenright): - """Show the calltip, bind events which will close it and reposition it. + """Show the call-tip, bind events which will close it and reposition it. + + text: the text to display in the call-tip + parenleft: index of the opening parenthesis in the text widget + parenright: index of the closing parenthesis in the text widget, + or the end of the line if there is no closing parenthesis """ - # Only called in CallTips, where lines are truncated + # Only called in calltip.Calltip, where lines are truncated self.text = text if self.tipwindow or not self.text: return - self.widget.mark_set(MARK_RIGHT, parenright) + self.anchor_widget.mark_set(MARK_RIGHT, parenright) self.parenline, self.parencol = map( - int, self.widget.index(parenleft).split(".")) + int, self.anchor_widget.index(parenleft).split(".")) - self.tipwindow = tw = Toplevel(self.widget) - self.position_window() - # remove border on calltip window - tw.wm_overrideredirect(1) - try: - # This command is only needed and available on Tk >= 8.4.0 for OSX - # Without it, call tips intrude on the typing process by grabbing - # the focus. - tw.tk.call("::tk::unsupported::MacWindowStyle", "style", tw._w, - "help", "noActivates") - except TclError: - pass - self.label = Label(tw, text=self.text, justify=LEFT, + super(CalltipWindow, self).showtip() + + self._bind_events() + + def showcontents(self): + """Create the call-tip widget.""" + self.label = Label(self.tipwindow, text=self.text, justify=LEFT, background="#ffffe0", relief=SOLID, borderwidth=1, - font = self.widget['font']) + font=self.anchor_widget['font']) self.label.pack() - tw.lift() # work around bug in Tk 8.5.18+ (issue #24570) - - self.checkhideid = self.widget.bind(CHECKHIDE_VIRTUAL_EVENT_NAME, - self.checkhide_event) - for seq in CHECKHIDE_SEQUENCES: - self.widget.event_add(CHECKHIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.after(CHECKHIDE_TIME, self.checkhide_event) - self.hideid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, - self.hide_event) - for seq in HIDE_SEQUENCES: - self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq) def checkhide_event(self, event=None): + """Handle CHECK_HIDE_EVENT: call hidetip or reschedule.""" if not self.tipwindow: - # If the event was triggered by the same event that unbinded + # If the event was triggered by the same event that unbound # this function, the function will be called nevertheless, # so do nothing in this case. return None - curline, curcol = map(int, self.widget.index("insert").split('.')) + + # Hide the call-tip if the insertion cursor moves outside of the + # parenthesis. + curline, curcol = map(int, self.anchor_widget.index("insert").split('.')) if curline < self.parenline or \ (curline == self.parenline and curcol <= self.parencol) or \ - self.widget.compare("insert", ">", MARK_RIGHT): + self.anchor_widget.compare("insert", ">", MARK_RIGHT): self.hidetip() return "break" - else: - self.position_window() - if self.checkhide_after_id is not None: - self.widget.after_cancel(self.checkhide_after_id) - self.checkhide_after_id = \ - self.widget.after(CHECKHIDE_TIME, self.checkhide_event) - return None + + # Not hiding the call-tip. + + self.position_window() + # Re-schedule this function to be called again in a short while. + if self.checkhide_after_id is not None: + self.anchor_widget.after_cancel(self.checkhide_after_id) + self.checkhide_after_id = \ + self.anchor_widget.after(CHECKHIDE_TIME, self.checkhide_event) + return None def hide_event(self, event): + """Handle HIDE_EVENT by calling hidetip.""" if not self.tipwindow: # See the explanation in checkhide_event. return None @@ -112,53 +120,81 @@ return "break" def hidetip(self): + """Hide the call-tip.""" if not self.tipwindow: return - for seq in CHECKHIDE_SEQUENCES: - self.widget.event_delete(CHECKHIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(CHECKHIDE_VIRTUAL_EVENT_NAME, self.checkhideid) - self.checkhideid = None - for seq in HIDE_SEQUENCES: - self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideid) - self.hideid = None - - self.label.destroy() + try: + self.label.destroy() + except TclError: + pass self.label = None - self.tipwindow.destroy() - self.tipwindow = None - self.widget.mark_unset(MARK_RIGHT) self.parenline = self.parencol = self.lastline = None + try: + self.anchor_widget.mark_unset(MARK_RIGHT) + except TclError: + pass - def is_active(self): - return bool(self.tipwindow) + try: + self._unbind_events() + except (TclError, ValueError): + # ValueError may be raised by MultiCall + pass + + super(CalltipWindow, self).hidetip() + + def _bind_events(self): + """Bind event handlers.""" + self.checkhideid = self.anchor_widget.bind(CHECKHIDE_EVENT, + self.checkhide_event) + for seq in CHECKHIDE_SEQUENCES: + self.anchor_widget.event_add(CHECKHIDE_EVENT, seq) + self.anchor_widget.after(CHECKHIDE_TIME, self.checkhide_event) + self.hideid = self.anchor_widget.bind(HIDE_EVENT, + self.hide_event) + for seq in HIDE_SEQUENCES: + self.anchor_widget.event_add(HIDE_EVENT, seq) + + def _unbind_events(self): + """Unbind event handlers.""" + for seq in CHECKHIDE_SEQUENCES: + self.anchor_widget.event_delete(CHECKHIDE_EVENT, seq) + self.anchor_widget.unbind(CHECKHIDE_EVENT, self.checkhideid) + self.checkhideid = None + for seq in HIDE_SEQUENCES: + self.anchor_widget.event_delete(HIDE_EVENT, seq) + self.anchor_widget.unbind(HIDE_EVENT, self.hideid) + self.hideid = None def _calltip_window(parent): # htest # from tkinter import Toplevel, Text, LEFT, BOTH top = Toplevel(parent) - top.title("Test calltips") + top.title("Test call-tips") x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("200x100+%d+%d" % (x + 250, y + 175)) + top.geometry("250x100+%d+%d" % (x + 175, y + 150)) text = Text(top) text.pack(side=LEFT, fill=BOTH, expand=1) text.insert("insert", "string.split") top.update() - calltip = CallTip(text) + calltip = CalltipWindow(text) def calltip_show(event): - calltip.showtip("(s=Hello world)", "insert", "end") + calltip.showtip("(s='Hello world')", "insert", "end") def calltip_hide(event): calltip.hidetip() text.event_add("<>", "(") text.event_add("<>", ")") text.bind("<>", calltip_show) text.bind("<>", calltip_hide) + text.focus_set() -if __name__=='__main__': +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_calltip_w', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_calltip_window) diff -Nru python3.7-3.7.0/Lib/idlelib/codecontext.py python3.7-3.7.1/Lib/idlelib/codecontext.py --- python3.7-3.7.0/Lib/idlelib/codecontext.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/codecontext.py 2018-10-20 06:04:19.000000000 +0000 @@ -233,6 +233,8 @@ CodeContext.reload() -if __name__ == "__main__": # pragma: no cover - import unittest - unittest.main('idlelib.idle_test.test_codecontext', verbosity=2, exit=False) +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_codecontext', verbosity=2, exit=False) + + # Add htest. diff -Nru python3.7-3.7.0/Lib/idlelib/colorizer.py python3.7-3.7.1/Lib/idlelib/colorizer.py --- python3.7-3.7.0/Lib/idlelib/colorizer.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/colorizer.py 2018-10-20 06:04:19.000000000 +0000 @@ -31,11 +31,12 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -def color_config(text): # Called from htest, Editor, and Turtle Demo. - '''Set color opitons of Text widget. +def color_config(text): + """Set color options of Text widget. - Should be called whenever ColorDelegator is called. - ''' + If ColorDelegator is used, this should be called first. + """ + # Called from htest, TextFrame, Editor, and Turtledemo. # Not automatic because ColorDelegator does not know 'text'. theme = idleConf.CurrentTheme() normal_colors = idleConf.GetHighlight(theme, 'normal') @@ -50,6 +51,7 @@ inactiveselectbackground=select_colors['background'], # new in 8.5 ) + class ColorDelegator(Delegator): def __init__(self): @@ -285,10 +287,10 @@ d = ColorDelegator() p.insertfilter(d) + if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_colorizer', - verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_colorizer', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_color_delegator) diff -Nru python3.7-3.7.0/Lib/idlelib/configdialog.py python3.7-3.7.1/Lib/idlelib/configdialog.py --- python3.7-3.7.0/Lib/idlelib/configdialog.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/configdialog.py 2018-10-20 06:04:19.000000000 +0000 @@ -30,10 +30,12 @@ from idlelib.codecontext import CodeContext from idlelib.parenmatch import ParenMatch from idlelib.paragraph import FormatParagraph +from idlelib.squeezer import Squeezer changes = ConfigChanges() # Reload changed options in the following classes. -reloadables = (AutoComplete, CodeContext, ParenMatch, FormatParagraph) +reloadables = (AutoComplete, CodeContext, ParenMatch, FormatParagraph, + Squeezer) class ConfigDialog(Toplevel): @@ -191,6 +193,7 @@ def destroy(self): global font_sample_text font_sample_text = self.fontpage.font_sample.get('1.0', 'end') + self.grab_release() super().destroy() def help(self): @@ -1747,9 +1750,9 @@ self.customlist.SetMenu(item_list, item_list[0]) # Revert to default key set. self.keyset_source.set(idleConf.defaultCfg['main'] - .Get('Keys', 'default')) + .Get('Keys', 'default')) self.builtin_name.set(idleConf.defaultCfg['main'].Get('Keys', 'name') - or idleConf.default_keys()) + or idleConf.default_keys()) # User can't back out of these changes, they must be applied now. changes.save_all() self.cd.save_all_changed_extensions() @@ -1816,6 +1819,10 @@ frame_context: Frame context_title: Label (*)context_int: Entry - context_lines + frame_shell: LabelFrame + frame_auto_squeeze_min_lines: Frame + auto_squeeze_min_lines_title: Label + (*)auto_squeeze_min_lines_int: Entry - auto_squeeze_min_lines frame_help: LabelFrame frame_helplist: Frame frame_helplist_buttons: Frame @@ -1841,6 +1848,9 @@ self.paren_bell = tracers.add( BooleanVar(self), ('extensions', 'ParenMatch', 'bell')) + self.auto_squeeze_min_lines = tracers.add( + StringVar(self), ('main', 'PyShell', 'auto-squeeze-min-lines')) + self.autosave = tracers.add( IntVar(self), ('main', 'General', 'autosave')) self.format_width = tracers.add( @@ -1854,8 +1864,10 @@ text=' Window Preferences') frame_editor = LabelFrame(self, borderwidth=2, relief=GROOVE, text=' Editor Preferences') + frame_shell = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Shell Preferences') frame_help = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' Additional Help Sources ') + text=' Additional Help Sources ') # Frame_window. frame_run = Frame(frame_window, borderwidth=0) startup_title = Label(frame_run, text='At Startup') @@ -1917,6 +1929,13 @@ self.context_int = Entry( frame_context, textvariable=self.context_lines, width=3) + # Frame_shell. + frame_auto_squeeze_min_lines = Frame(frame_shell, borderwidth=0) + auto_squeeze_min_lines_title = Label(frame_auto_squeeze_min_lines, + text='Auto-Squeeze Min. Lines:') + self.auto_squeeze_min_lines_int = Entry( + frame_auto_squeeze_min_lines, width=4, + textvariable=self.auto_squeeze_min_lines) # frame_help. frame_helplist = Frame(frame_help) @@ -1942,6 +1961,7 @@ # Body. frame_window.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_editor.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_shell.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) # frame_run. frame_run.pack(side=TOP, padx=5, pady=0, fill=X) @@ -1982,6 +2002,11 @@ context_title.pack(side=LEFT, anchor=W, padx=5, pady=5) self.context_int.pack(side=TOP, padx=5, pady=5) + # frame_auto_squeeze_min_lines + frame_auto_squeeze_min_lines.pack(side=TOP, padx=5, pady=0, fill=X) + auto_squeeze_min_lines_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.auto_squeeze_min_lines_int.pack(side=TOP, padx=5, pady=5) + # frame_help. frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) @@ -2017,6 +2042,10 @@ self.context_lines.set(idleConf.GetOption( 'extensions', 'CodeContext', 'maxlines', type='int')) + # Set variables for shell windows. + self.auto_squeeze_min_lines.set(idleConf.GetOption( + 'main', 'PyShell', 'auto-squeeze-min-lines', type='int')) + # Set additional help sources. self.user_helplist = idleConf.GetAllExtraHelpSourcesList() self.helplist.delete(0, 'end') @@ -2210,6 +2239,9 @@ CodeContext: Maxlines is the maximum number of code context lines to display when Code Context is turned on for an editor window. + +Shell Preferences: Auto-Squeeze Min. Lines is the minimum number of lines +of output to automatically "squeeze". ''' } @@ -2269,8 +2301,8 @@ if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_configdialog', - verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_configdialog', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(ConfigDialog) diff -Nru python3.7-3.7.0/Lib/idlelib/config_key.py python3.7-3.7.1/Lib/idlelib/config_key.py --- python3.7-3.7.0/Lib/idlelib/config_key.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/config_key.py 2018-10-20 06:04:19.000000000 +0000 @@ -235,10 +235,12 @@ return if (self.advanced or self.KeysOK(keys)) and self.bind_ok(keys): self.result = keys + self.grab_release() self.destroy() def Cancel(self, event=None): self.result='' + self.grab_release() self.destroy() def KeysOK(self, keys): @@ -291,8 +293,8 @@ if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(GetKeysDialog) diff -Nru python3.7-3.7.0/Lib/idlelib/config-main.def python3.7-3.7.1/Lib/idlelib/config-main.def --- python3.7-3.7.0/Lib/idlelib/config-main.def 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/config-main.def 2018-10-20 06:04:19.000000000 +0000 @@ -66,6 +66,9 @@ font-bold= 0 encoding= none +[PyShell] +auto-squeeze-min-lines= 50 + [Indent] use-spaces= 1 num-spaces= 4 diff -Nru python3.7-3.7.0/Lib/idlelib/config.py python3.7-3.7.1/Lib/idlelib/config.py --- python3.7-3.7.0/Lib/idlelib/config.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/config.py 2018-10-20 06:04:19.000000000 +0000 @@ -925,7 +925,7 @@ print('\nlines = ', line, ', crc = ', crc, sep='') if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_config', - verbosity=2, exit=False) - #_dump() + from unittest import main + main('idlelib.idle_test.test_config', verbosity=2, exit=False) + + # Run revised _dump() as htest? diff -Nru python3.7-3.7.0/Lib/idlelib/debugger.py python3.7-3.7.1/Lib/idlelib/debugger.py --- python3.7-3.7.0/Lib/idlelib/debugger.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/debugger.py 2018-10-20 06:04:19.000000000 +0000 @@ -6,13 +6,13 @@ from idlelib import macosx from idlelib.scrolledlist import ScrolledList -from idlelib.windows import ListedToplevel +from idlelib.window import ListedToplevel class Idb(bdb.Bdb): def __init__(self, gui): - self.gui = gui + self.gui = gui # An instance of Debugger or proxy of remote. bdb.Bdb.__init__(self) def user_line(self, frame): @@ -40,7 +40,7 @@ prev_name = prev_frame.f_code.co_filename if 'idlelib' in prev_name and 'debugger' in prev_name: # catch both idlelib/debugger.py and idlelib/debugger_r.py - # on both posix and windows + # on both Posix and Windows return False return self.in_rpc_code(prev_frame) @@ -63,7 +63,7 @@ if idb is None: idb = Idb(self) self.pyshell = pyshell - self.idb = idb + self.idb = idb # If passed, a proxy of remote instance. self.frame = None self.make_gui() self.interacting = 0 @@ -542,3 +542,9 @@ def close(self): self.frame.destroy() + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_debugger', verbosity=2, exit=False) + +# TODO: htest? diff -Nru python3.7-3.7.0/Lib/idlelib/debugger_r.py python3.7-3.7.1/Lib/idlelib/debugger_r.py --- python3.7-3.7.0/Lib/idlelib/debugger_r.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/debugger_r.py 2018-10-20 06:04:19.000000000 +0000 @@ -386,3 +386,8 @@ idb_adap_oid_ret = rpcclt.remotecall("exec", "start_the_debugger",\ (gui_adap_oid,), {}) assert idb_adap_oid_ret == idb_adap_oid, 'Idb restarted with different oid' + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_debugger', verbosity=2, exit=False) diff -Nru python3.7-3.7.0/Lib/idlelib/debugobj.py python3.7-3.7.1/Lib/idlelib/debugobj.py --- python3.7-3.7.0/Lib/idlelib/debugobj.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/debugobj.py 2018-10-20 06:04:19.000000000 +0000 @@ -71,7 +71,7 @@ class AtomicObjectTreeItem(ObjectTreeItem): def IsExpandable(self): - return 0 + return False class SequenceTreeItem(ObjectTreeItem): def IsExpandable(self): @@ -135,5 +135,8 @@ node.update() if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_debugobj', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_object_browser) diff -Nru python3.7-3.7.0/Lib/idlelib/debugobj_r.py python3.7-3.7.1/Lib/idlelib/debugobj_r.py --- python3.7-3.7.0/Lib/idlelib/debugobj_r.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/debugobj_r.py 2018-10-20 06:04:19.000000000 +0000 @@ -34,3 +34,8 @@ def _GetSubList(self): sub_list = self.sockio.remotecall(self.oid, "_GetSubList", (), {}) return [StubObjectTreeItem(self.sockio, oid) for oid in sub_list] + + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_debugobj_r', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/editor.py python3.7-3.7.1/Lib/idlelib/editor.py --- python3.7-3.7.0/Lib/idlelib/editor.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/editor.py 2018-10-20 06:04:19.000000000 +0000 @@ -2,9 +2,7 @@ import importlib.util import os import platform -import re import string -import sys import tokenize import traceback import webbrowser @@ -25,7 +23,7 @@ from idlelib import query from idlelib import replace from idlelib import search -from idlelib import windows +from idlelib import window # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 @@ -50,15 +48,15 @@ from idlelib.undo import UndoDelegator from idlelib.iomenu import IOBinding, encoding from idlelib import mainmenu - from tkinter import Toplevel, EventType from idlelib.statusbar import MultiStatusBar from idlelib.autocomplete import AutoComplete from idlelib.autoexpand import AutoExpand - from idlelib.calltips import CallTips + from idlelib.calltip import Calltip from idlelib.codecontext import CodeContext from idlelib.paragraph import FormatParagraph from idlelib.parenmatch import ParenMatch - from idlelib.rstrip import RstripExtension + from idlelib.rstrip import Rstrip + from idlelib.squeezer import Squeezer from idlelib.zoomheight import ZoomHeight filesystemencoding = sys.getfilesystemencoding() # for file names @@ -101,7 +99,7 @@ root = root or flist.root self.root = root self.menubar = Menu(root) - self.top = top = windows.ListedToplevel(root, menu=self.menubar) + self.top = top = window.ListedToplevel(root, menu=self.menubar) if flist: self.tkinter_vars = flist.vars #self.top.instance_dict makes flist.inversedict available to @@ -138,7 +136,7 @@ self.top.protocol("WM_DELETE_WINDOW", self.close) self.top.bind("<>", self.close_event) if macosx.isAquaTk(): - # Command-W on editorwindows doesn't work without this. + # Command-W on editor windows doesn't work without this. text.bind('<>', self.close_event) # Some OS X systems have only one mouse button, so use # control-click for popup context menus there. For two @@ -267,7 +265,7 @@ self.saved_change_hook() self.update_recent_files_list() self.load_extensions() - menu = self.menudict.get('windows') + menu = self.menudict.get('window') if menu: end = menu.index("end") if end is None: @@ -276,7 +274,7 @@ menu.add_separator() end = end + 1 self.wmenu_end = end - windows.register_callback(self.postwindowsmenu) + window.register_callback(self.postwindowsmenu) # Some abstractions so IDLE extensions are cross-IDE self.askyesno = tkMessageBox.askyesno @@ -310,15 +308,18 @@ scriptbinding = ScriptBinding(self) text.bind("<>", scriptbinding.check_module_event) text.bind("<>", scriptbinding.run_module_event) - text.bind("<>", self.RstripExtension(self).do_rstrip) - calltips = self.CallTips(self) - text.bind("<>", calltips.try_open_calltip_event) - #refresh-calltips must come after paren-closed to work right - text.bind("<>", calltips.refresh_calltip_event) - text.bind("<>", calltips.force_open_calltip_event) + text.bind("<>", self.Rstrip(self).do_rstrip) + ctip = self.Calltip(self) + text.bind("<>", ctip.try_open_calltip_event) + #refresh-calltip must come after paren-closed to work right + text.bind("<>", ctip.refresh_calltip_event) + text.bind("<>", ctip.force_open_calltip_event) text.bind("<>", self.ZoomHeight(self).zoom_height_event) text.bind("<>", self.CodeContext(self).toggle_code_context_event) + squeezer = self.Squeezer(self) + text.bind("<>", + squeezer.squeeze_current_text_event) def _filename_to_unicode(self, filename): """Return filename as BMP unicode so diplayable in Tk.""" @@ -410,7 +411,7 @@ ("format", "F_ormat"), ("run", "_Run"), ("options", "_Options"), - ("windows", "_Window"), + ("window", "_Window"), ("help", "_Help"), ] @@ -436,14 +437,14 @@ self.reset_help_menu_entries() def postwindowsmenu(self): - # Only called when Windows menu exists - menu = self.menudict['windows'] + # Only called when Window menu exists + menu = self.menudict['window'] end = menu.index("end") if end is None: end = -1 if end > self.wmenu_end: menu.delete(self.wmenu_end+1, end) - windows.add_windows_to_menu(menu) + window.add_windows_to_menu(menu) def handle_yview(self, event, *args): "Handle scrollbar." @@ -457,12 +458,19 @@ return 'break' def mousescroll(self, event): - "Handle scroll wheel." - up = {EventType.MouseWheel: event.delta >= 0 == darwin, + """Handle scrollwheel event. + + For wheel up, event.delta = 120*n on Windows, -1*n on darwin, + where n can be > 1 if one scrolls fast. Flicking the wheel + generates up to maybe 20 events with n up to 10 or more 1. + Macs use wheel down (delta = 1*n) to scroll up, so positive + delta means to scroll up on both systems. + + X-11 sends Control-Button-4 event instead. + """ + up = {EventType.MouseWheel: event.delta > 0, EventType.Button: event.num == 4} - lines = 5 - if up[event.type]: - lines = -lines + lines = -5 if up[event.type] else 5 self.text.yview_scroll(lines, 'units') return 'break' @@ -1012,7 +1020,7 @@ def _close(self): if self.io.filename: self.update_recent_files_list(new_file=self.io.filename) - windows.unregister_callback(self.postwindowsmenu) + window.unregister_callback(self.postwindowsmenu) self.unload_extensions() self.io.close() self.io = None @@ -1701,13 +1709,17 @@ filename = None macosx.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) - edit.text.bind("<>", edit.close_event) + text = edit.text + text['height'] = 10 + for i in range(20): + text.insert('insert', ' '*i + str(i) + '\n') + # text.bind("<>", edit.close_event) # Does not stop error, neither does following # edit.text.bind("<>", edit.close_event) if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_editor', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_editor', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_editor_window) diff -Nru python3.7-3.7.0/Lib/idlelib/filelist.py python3.7-3.7.1/Lib/idlelib/filelist.py --- python3.7-3.7.0/Lib/idlelib/filelist.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/filelist.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,7 +1,7 @@ -import os +"idlelib.filelist" -from tkinter import * -import tkinter.messagebox as tkMessageBox +import os +from tkinter import messagebox as tkMessageBox class FileList: @@ -111,7 +111,8 @@ return os.path.normpath(filename) -def _test(): +def _test(): # TODO check and convert to htest + from tkinter import Tk from idlelib.editor import fixwordbreaks from idlelib.run import fix_scaling import sys @@ -120,13 +121,12 @@ fixwordbreaks(root) root.withdraw() flist = FileList(root) - if sys.argv[1:]: - for filename in sys.argv[1:]: - flist.open(filename) - else: - flist.new() + flist.new() if flist.inversedict: root.mainloop() if __name__ == '__main__': - _test() + from unittest import main + main('idlelib.idle_test.test_filelist', verbosity=2) + +# _test() diff -Nru python3.7-3.7.0/Lib/idlelib/grep.py python3.7-3.7.1/Lib/idlelib/grep.py --- python3.7-3.7.0/Lib/idlelib/grep.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/grep.py 2018-10-20 06:04:19.000000000 +0000 @@ -193,8 +193,8 @@ button.pack() if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_grep', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_grep', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_grep_dialog) diff -Nru python3.7-3.7.0/Lib/idlelib/help_about.py python3.7-3.7.1/Lib/idlelib/help_about.py --- python3.7-3.7.0/Lib/idlelib/help_about.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/help_about.py 2018-10-20 06:04:19.000000000 +0000 @@ -195,11 +195,13 @@ def ok(self, event=None): "Dismiss help_about dialog." + self.grab_release() self.destroy() if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_help_about', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_help_about', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(AboutDialog) diff -Nru python3.7-3.7.0/Lib/idlelib/help.py python3.7-3.7.1/Lib/idlelib/help.py --- python3.7-3.7.0/Lib/idlelib/help.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/help.py 2018-10-20 06:04:19.000000000 +0000 @@ -271,5 +271,8 @@ HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version()) if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_help', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(show_idlehelp) diff -Nru python3.7-3.7.0/Lib/idlelib/hyperparser.py python3.7-3.7.1/Lib/idlelib/hyperparser.py --- python3.7-3.7.0/Lib/idlelib/hyperparser.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/hyperparser.py 2018-10-20 06:04:19.000000000 +0000 @@ -308,5 +308,5 @@ if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_hyperparser', verbosity=2) + from unittest import main + main('idlelib.idle_test.test_hyperparser', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/htest.py python3.7-3.7.1/Lib/idlelib/idle_test/htest.py --- python3.7-3.7.0/Lib/idlelib/idle_test/htest.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/htest.py 2018-10-20 06:04:19.000000000 +0000 @@ -65,6 +65,7 @@ outwin.OutputWindow (indirectly being tested with grep test) ''' +import idlelib.pyshell # Set Windows DPI awareness before Tk(). from importlib import import_module import tkinter as tk from tkinter.ttk import Scrollbar @@ -79,11 +80,14 @@ "are correctly displayed.\n [Close] to exit.", } +# TODO implement ^\; adding '' to function does not work. _calltip_window_spec = { 'file': 'calltip_w', 'kwds': {}, 'msg': "Typing '(' should display a calltip.\n" "Typing ') should hide the calltip.\n" + "So should moving cursor out of argument area.\n" + "Force-open-calltip does not work here.\n" } _module_browser_spec = { @@ -159,7 +163,7 @@ 'msg': "Click the 'Show GrepDialog' button.\n" "Test the various 'Find-in-files' functions.\n" "The results should be displayed in a new '*Output*' window.\n" - "'Right-click'->'Goto file/line' anywhere in the search results " + "'Right-click'->'Go to file/line' anywhere in the search results " "should open that file \nin a new EditorWindow." } @@ -296,16 +300,6 @@ "Check that exc_value, exc_tb, and exc_type are correct.\n" } -_tabbed_pages_spec = { - 'file': 'tabbedpages', - 'kwds': {}, - 'msg': "Toggle between the two tabs 'foo' and 'bar'\n" - "Add a tab by entering a suitable name for it.\n" - "Remove an existing tab by entering its name.\n" - "Remove all existing tabs.\n" - " is an invalid add page and remove page name.\n" - } - _tooltip_spec = { 'file': 'tooltip', 'kwds': {}, diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/README.txt python3.7-3.7.1/Lib/idlelib/idle_test/README.txt --- python3.7-3.7.0/Lib/idlelib/idle_test/README.txt 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/README.txt 2018-10-20 06:04:19.000000000 +0000 @@ -15,28 +15,27 @@ 1. Test Files The idle directory, idlelib, has over 60 xyz.py files. The idle_test -subdirectory should contain a test_xyz.py for each, where 'xyz' is -lowercased even if xyz.py is not. Here is a possible template, with the -blanks after '.' and 'as', and before and after '_' to be filled in. +subdirectory contains test_xyz.py for each implementation file xyz.py. +To add a test for abc.py, open idle_test/template.py and immediately +Save As test_abc.py. Insert 'abc' on the first line, and replace +'zzdummy' with 'abc. + +Remove the imports of requires and tkinter if not needed. Otherwise, +add to the tkinter imports as needed. + +Add a prefix to 'Test' for the initial test class. The template class +contains code needed or possibly needed for gui tests. See the next +section if doing gui tests. If not, and not needed for further classes, +this code can be removed. -import unittest -from test.support import requires -import idlelib. as - -class _Test(unittest.TestCase): - - def test_(self): - -if __name__ == '__main__': - unittest.main(verbosity=2) - -Add the following at the end of xyy.py, with the appropriate name added -after 'test_'. Some files already have something like this for htest. -If so, insert the import and unittest.main lines before the htest lines. +Add the following at the end of abc.py. If an htest was added first, +insert the import and main lines before the htest lines. if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_abc', verbosity=2, exit=False) + +The ', exit=False' is only needed if an htest follows. @@ -55,12 +54,14 @@ requires('gui') To guard a test class, put "requires('gui')" in its setUpClass function. +The template.py file does this. -To avoid interfering with other GUI tests, all GUI objects must be destroyed and -deleted by the end of the test. The Tk root created in a setUpX function should -be destroyed in the corresponding tearDownX and the module or class attribute -deleted. Others widgets should descend from the single root and the attributes -deleted BEFORE root is destroyed. See https://bugs.python.org/issue20567. +To avoid interfering with other GUI tests, all GUI objects must be +destroyed and deleted by the end of the test. The Tk root created in a +setUpX function should be destroyed in the corresponding tearDownX and +the module or class attribute deleted. Others widgets should descend +from the single root and the attributes deleted BEFORE root is +destroyed. See https://bugs.python.org/issue20567. @classmethod def setUpClass(cls): @@ -75,12 +76,23 @@ cls.root.destroy() del cls.root -The update_idletasks call is sometimes needed to prevent the following warning -either when running a test alone or as part of the test suite (#27196). +The update_idletasks call is sometimes needed to prevent the following +warning either when running a test alone or as part of the test suite +(#27196). It should not hurt if not needed. + can't invoke "event" command: application has been destroyed ... "ttk::ThemeChanged" +If a test creates instance 'e' of EditorWindow, call 'e._close()' before +or as the first part of teardown. The effect of omitting this depends +on the later shutdown. Then enable the after_cancel loop in the +template. This prevents messages like the following. + +bgerror failed to handle background error. + Original error: invalid command name "106096696timer_event" + Error in bgerror: can't invoke "tk" command: application has been destroyed + Requires('gui') causes the test(s) it guards to be skipped if any of these conditions are met: diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/template.py python3.7-3.7.1/Lib/idlelib/idle_test/template.py --- python3.7-3.7.0/Lib/idlelib/idle_test/template.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/template.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,30 @@ +"Test , coverage %." + +from idlelib import zzdummy +import unittest +from test.support import requires +from tkinter import Tk + + +class Test(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.update_idletasks() +## for id in cls.root.tk.call('after', 'info'): +## cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_init(self): + self.assertTrue(True) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_autocomplete.py python3.7-3.7.1/Lib/idlelib/idle_test/test_autocomplete.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_autocomplete.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_autocomplete.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,7 +1,5 @@ -''' Test autocomplete and autocomple_w +"Test autocomplete, coverage 57%." -Coverage of autocomple: 56% -''' import unittest from test.support import requires from tkinter import Tk, Text @@ -11,9 +9,6 @@ from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Event -class AutoCompleteWindow: - def complete(): - return class DummyEditwin: def __init__(self, root, text): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_autocomplete_w.py python3.7-3.7.1/Lib/idlelib/idle_test/test_autocomplete_w.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_autocomplete_w.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_autocomplete_w.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,32 @@ +"Test autocomplete_w, coverage 11%." + +import unittest +from test.support import requires +from tkinter import Tk, Text + +import idlelib.autocomplete_w as acw + + +class AutoCompleteWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.acw = acw.AutoCompleteWindow(cls.text) + + @classmethod + def tearDownClass(cls): + del cls.text, cls.acw + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_init(self): + self.assertEqual(self.acw.widget, self.text) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_autoexpand.py python3.7-3.7.1/Lib/idlelib/idle_test/test_autoexpand.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_autoexpand.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_autoexpand.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,9 +1,9 @@ -"""Unit tests for idlelib.autoexpand""" +"Test autoexpand, coverage 100%." + +from idlelib.autoexpand import AutoExpand import unittest from test.support import requires from tkinter import Text, Tk -#from idlelib.idle_test.mock_tk import Text -from idlelib.autoexpand import AutoExpand class Dummy_Editwin: @@ -15,15 +15,27 @@ @classmethod def setUpClass(cls): - if 'tkinter' in str(Text): - requires('gui') - cls.tk = Tk() - cls.text = Text(cls.tk) - else: - cls.text = Text() + requires('gui') + cls.tk = Tk() + cls.text = Text(cls.tk) cls.auto_expand = AutoExpand(Dummy_Editwin(cls.text)) cls.auto_expand.bell = lambda: None +# If mock_tk.Text._decode understood indexes 'insert' with suffixed 'linestart', +# 'wordstart', and 'lineend', used by autoexpand, we could use the following +# to run these test on non-gui machines (but check bell). +## try: +## requires('gui') +## #raise ResourceDenied() # Uncomment to test mock. +## except ResourceDenied: +## from idlelib.idle_test.mock_tk import Text +## cls.text = Text() +## cls.text.bell = lambda: None +## else: +## from tkinter import Tk, Text +## cls.tk = Tk() +## cls.text = Text(cls.tk) + @classmethod def tearDownClass(cls): del cls.text, cls.auto_expand diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_browser.py python3.7-3.7.1/Lib/idlelib/idle_test/test_browser.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_browser.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_browser.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,20 +1,16 @@ -""" Test idlelib.browser. +"Test browser, coverage 90%." -Coverage: 88% -(Higher, because should exclude 3 lines that .coveragerc won't exclude.) -""" +from idlelib import browser +from test.support import requires +import unittest +from unittest import mock +from idlelib.idle_test.mock_idle import Func from collections import deque import os.path import pyclbr from tkinter import Tk -from test.support import requires -import unittest -from unittest import mock -from idlelib.idle_test.mock_idle import Func - -from idlelib import browser from idlelib import filelist from idlelib.tree import TreeNode diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_calltip.py python3.7-3.7.1/Lib/idlelib/idle_test/test_calltip.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_calltip.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_calltip.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,218 @@ +"Test calltip, coverage 60%" + +from idlelib import calltip +import unittest +import textwrap +import types + +default_tip = calltip._default_callable_argspec + + +# Test Class TC is used in multiple get_argspec test methods +class TC(): + 'doc' + tip = "(ai=None, *b)" + def __init__(self, ai=None, *b): 'doc' + __init__.tip = "(self, ai=None, *b)" + def t1(self): 'doc' + t1.tip = "(self)" + def t2(self, ai, b=None): 'doc' + t2.tip = "(self, ai, b=None)" + def t3(self, ai, *args): 'doc' + t3.tip = "(self, ai, *args)" + def t4(self, *args): 'doc' + t4.tip = "(self, *args)" + def t5(self, ai, b=None, *args, **kw): 'doc' + t5.tip = "(self, ai, b=None, *args, **kw)" + def t6(no, self): 'doc' + t6.tip = "(no, self)" + def __call__(self, ci): 'doc' + __call__.tip = "(self, ci)" + # attaching .tip to wrapped methods does not work + @classmethod + def cm(cls, a): 'doc' + @staticmethod + def sm(b): 'doc' + + +tc = TC() +signature = calltip.get_argspec # 2.7 and 3.x use different functions + + +class Get_signatureTest(unittest.TestCase): + # The signature function must return a string, even if blank. + # Test a variety of objects to be sure that none cause it to raise + # (quite aside from getting as correct an answer as possible). + # The tests of builtins may break if inspect or the docstrings change, + # but a red buildbot is better than a user crash (as has happened). + # For a simple mismatch, change the expected output to the actual. + + def test_builtins(self): + + # Python class that inherits builtin methods + class List(list): "List() doc" + + # Simulate builtin with no docstring for default tip test + class SB: __call__ = None + + def gtest(obj, out): + self.assertEqual(signature(obj), out) + + if List.__doc__ is not None: + gtest(List, '(iterable=(), /)' + calltip._argument_positional + + '\n' + List.__doc__) + gtest(list.__new__, + '(*args, **kwargs)\n' + 'Create and return a new object. ' + 'See help(type) for accurate signature.') + gtest(list.__init__, + '(self, /, *args, **kwargs)' + + calltip._argument_positional + '\n' + + 'Initialize self. See help(type(self)) for accurate signature.') + append_doc = (calltip._argument_positional + + "\nAppend object to the end of the list.") + gtest(list.append, '(self, object, /)' + append_doc) + gtest(List.append, '(self, object, /)' + append_doc) + gtest([].append, '(object, /)' + append_doc) + + gtest(types.MethodType, "method(function, instance)") + gtest(SB(), default_tip) + import re + p = re.compile('') + gtest(re.sub, '''\ +(pattern, repl, string, count=0, flags=0) +Return the string obtained by replacing the leftmost +non-overlapping occurrences of the pattern in string by the +replacement repl. repl can be either a string or a callable; +if a string, backslash escapes in it are processed. If it is +a callable, it's passed the Match object and must return''') + gtest(p.sub, '''\ +(repl, string, count=0) +Return the string obtained by replacing the leftmost \ +non-overlapping occurrences o...''') + + def test_signature_wrap(self): + if textwrap.TextWrapper.__doc__ is not None: + self.assertEqual(signature(textwrap.TextWrapper), '''\ +(width=70, initial_indent='', subsequent_indent='', expand_tabs=True, + replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, + drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None, + placeholder=' [...]')''') + + def test_docline_truncation(self): + def f(): pass + f.__doc__ = 'a'*300 + self.assertEqual(signature(f), '()\n' + 'a' * (calltip._MAX_COLS-3) + '...') + + def test_multiline_docstring(self): + # Test fewer lines than max. + self.assertEqual(signature(range), + "range(stop) -> range object\n" + "range(start, stop[, step]) -> range object") + + # Test max lines + self.assertEqual(signature(bytes), '''\ +bytes(iterable_of_ints) -> bytes +bytes(string, encoding[, errors]) -> bytes +bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer +bytes(int) -> bytes object of size given by the parameter initialized with null bytes +bytes() -> empty bytes object''') + + # Test more than max lines + def f(): pass + f.__doc__ = 'a\n' * 15 + self.assertEqual(signature(f), '()' + '\na' * calltip._MAX_LINES) + + def test_functions(self): + def t1(): 'doc' + t1.tip = "()" + def t2(a, b=None): 'doc' + t2.tip = "(a, b=None)" + def t3(a, *args): 'doc' + t3.tip = "(a, *args)" + def t4(*args): 'doc' + t4.tip = "(*args)" + def t5(a, b=None, *args, **kw): 'doc' + t5.tip = "(a, b=None, *args, **kw)" + + doc = '\ndoc' if t1.__doc__ is not None else '' + for func in (t1, t2, t3, t4, t5, TC): + self.assertEqual(signature(func), func.tip + doc) + + def test_methods(self): + doc = '\ndoc' if TC.__doc__ is not None else '' + for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__): + self.assertEqual(signature(meth), meth.tip + doc) + self.assertEqual(signature(TC.cm), "(a)" + doc) + self.assertEqual(signature(TC.sm), "(b)" + doc) + + def test_bound_methods(self): + # test that first parameter is correctly removed from argspec + doc = '\ndoc' if TC.__doc__ is not None else '' + for meth, mtip in ((tc.t1, "()"), (tc.t4, "(*args)"), + (tc.t6, "(self)"), (tc.__call__, '(ci)'), + (tc, '(ci)'), (TC.cm, "(a)"),): + self.assertEqual(signature(meth), mtip + doc) + + def test_starred_parameter(self): + # test that starred first parameter is *not* removed from argspec + class C: + def m1(*args): pass + c = C() + for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"),): + self.assertEqual(signature(meth), mtip) + + def test_invalid_method_signature(self): + class C: + def m2(**kwargs): pass + class Test: + def __call__(*, a): pass + + mtip = calltip._invalid_method + self.assertEqual(signature(C().m2), mtip) + self.assertEqual(signature(Test()), mtip) + + def test_non_ascii_name(self): + # test that re works to delete a first parameter name that + # includes non-ascii chars, such as various forms of A. + uni = "(A\u0391\u0410\u05d0\u0627\u0905\u1e00\u3042, a)" + assert calltip._first_param.sub('', uni) == '(a)' + + def test_no_docstring(self): + def nd(s): + pass + TC.nd = nd + self.assertEqual(signature(nd), "(s)") + self.assertEqual(signature(TC.nd), "(s)") + self.assertEqual(signature(tc.nd), "()") + + def test_attribute_exception(self): + class NoCall: + def __getattr__(self, name): + raise BaseException + class CallA(NoCall): + def __call__(oui, a, b, c): + pass + class CallB(NoCall): + def __call__(self, ci): + pass + + for meth, mtip in ((NoCall, default_tip), (CallA, default_tip), + (NoCall(), ''), (CallA(), '(a, b, c)'), + (CallB(), '(ci)')): + self.assertEqual(signature(meth), mtip) + + def test_non_callables(self): + for obj in (0, 0.0, '0', b'0', [], {}): + self.assertEqual(signature(obj), '') + + +class Get_entityTest(unittest.TestCase): + def test_bad_entity(self): + self.assertIsNone(calltip.get_entity('1/0')) + def test_good_entity(self): + self.assertIs(calltip.get_entity('int'), int) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_calltips.py python3.7-3.7.1/Lib/idlelib/idle_test/test_calltips.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_calltips.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_calltips.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,202 +0,0 @@ -import unittest -import idlelib.calltips as ct -import textwrap -import types - -default_tip = ct._default_callable_argspec - -# Test Class TC is used in multiple get_argspec test methods -class TC(): - 'doc' - tip = "(ai=None, *b)" - def __init__(self, ai=None, *b): 'doc' - __init__.tip = "(self, ai=None, *b)" - def t1(self): 'doc' - t1.tip = "(self)" - def t2(self, ai, b=None): 'doc' - t2.tip = "(self, ai, b=None)" - def t3(self, ai, *args): 'doc' - t3.tip = "(self, ai, *args)" - def t4(self, *args): 'doc' - t4.tip = "(self, *args)" - def t5(self, ai, b=None, *args, **kw): 'doc' - t5.tip = "(self, ai, b=None, *args, **kw)" - def t6(no, self): 'doc' - t6.tip = "(no, self)" - def __call__(self, ci): 'doc' - __call__.tip = "(self, ci)" - # attaching .tip to wrapped methods does not work - @classmethod - def cm(cls, a): 'doc' - @staticmethod - def sm(b): 'doc' - -tc = TC() - -signature = ct.get_argspec # 2.7 and 3.x use different functions -class Get_signatureTest(unittest.TestCase): - # The signature function must return a string, even if blank. - # Test a variety of objects to be sure that none cause it to raise - # (quite aside from getting as correct an answer as possible). - # The tests of builtins may break if inspect or the docstrings change, - # but a red buildbot is better than a user crash (as has happened). - # For a simple mismatch, change the expected output to the actual. - - def test_builtins(self): - - # Python class that inherits builtin methods - class List(list): "List() doc" - - # Simulate builtin with no docstring for default tip test - class SB: __call__ = None - - def gtest(obj, out): - self.assertEqual(signature(obj), out) - - if List.__doc__ is not None: - gtest(List, '(iterable=(), /)' + ct._argument_positional + '\n' + - List.__doc__) - gtest(list.__new__, - '(*args, **kwargs)\nCreate and return a new object. See help(type) for accurate signature.') - gtest(list.__init__, - '(self, /, *args, **kwargs)' + ct._argument_positional + '\n' + - 'Initialize self. See help(type(self)) for accurate signature.') - append_doc = ct._argument_positional + "\nAppend object to the end of the list." - gtest(list.append, '(self, object, /)' + append_doc) - gtest(List.append, '(self, object, /)' + append_doc) - gtest([].append, '(object, /)' + append_doc) - - gtest(types.MethodType, "method(function, instance)") - gtest(SB(), default_tip) - import re - p = re.compile('') - gtest(re.sub, '''(pattern, repl, string, count=0, flags=0)\nReturn the string obtained by replacing the leftmost -non-overlapping occurrences of the pattern in string by the -replacement repl. repl can be either a string or a callable; -if a string, backslash escapes in it are processed. If it is -a callable, it's passed the Match object and must return''') - gtest(p.sub, '''(repl, string, count=0)\nReturn the string obtained by replacing the leftmost non-overlapping occurrences o...''') - - def test_signature_wrap(self): - if textwrap.TextWrapper.__doc__ is not None: - self.assertEqual(signature(textwrap.TextWrapper), '''\ -(width=70, initial_indent='', subsequent_indent='', expand_tabs=True, - replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, - drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None, - placeholder=' [...]')''') - - def test_docline_truncation(self): - def f(): pass - f.__doc__ = 'a'*300 - self.assertEqual(signature(f), '()\n' + 'a' * (ct._MAX_COLS-3) + '...') - - def test_multiline_docstring(self): - # Test fewer lines than max. - self.assertEqual(signature(range), - "range(stop) -> range object\n" - "range(start, stop[, step]) -> range object") - - # Test max lines - self.assertEqual(signature(bytes), '''\ -bytes(iterable_of_ints) -> bytes -bytes(string, encoding[, errors]) -> bytes -bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer -bytes(int) -> bytes object of size given by the parameter initialized with null bytes -bytes() -> empty bytes object''') - - # Test more than max lines - def f(): pass - f.__doc__ = 'a\n' * 15 - self.assertEqual(signature(f), '()' + '\na' * ct._MAX_LINES) - - def test_functions(self): - def t1(): 'doc' - t1.tip = "()" - def t2(a, b=None): 'doc' - t2.tip = "(a, b=None)" - def t3(a, *args): 'doc' - t3.tip = "(a, *args)" - def t4(*args): 'doc' - t4.tip = "(*args)" - def t5(a, b=None, *args, **kw): 'doc' - t5.tip = "(a, b=None, *args, **kw)" - - doc = '\ndoc' if t1.__doc__ is not None else '' - for func in (t1, t2, t3, t4, t5, TC): - self.assertEqual(signature(func), func.tip + doc) - - def test_methods(self): - doc = '\ndoc' if TC.__doc__ is not None else '' - for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__): - self.assertEqual(signature(meth), meth.tip + doc) - self.assertEqual(signature(TC.cm), "(a)" + doc) - self.assertEqual(signature(TC.sm), "(b)" + doc) - - def test_bound_methods(self): - # test that first parameter is correctly removed from argspec - doc = '\ndoc' if TC.__doc__ is not None else '' - for meth, mtip in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"), - (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),): - self.assertEqual(signature(meth), mtip + doc) - - def test_starred_parameter(self): - # test that starred first parameter is *not* removed from argspec - class C: - def m1(*args): pass - c = C() - for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"),): - self.assertEqual(signature(meth), mtip) - - def test_invalid_method_signature(self): - class C: - def m2(**kwargs): pass - class Test: - def __call__(*, a): pass - - mtip = ct._invalid_method - self.assertEqual(signature(C().m2), mtip) - self.assertEqual(signature(Test()), mtip) - - def test_non_ascii_name(self): - # test that re works to delete a first parameter name that - # includes non-ascii chars, such as various forms of A. - uni = "(A\u0391\u0410\u05d0\u0627\u0905\u1e00\u3042, a)" - assert ct._first_param.sub('', uni) == '(a)' - - def test_no_docstring(self): - def nd(s): - pass - TC.nd = nd - self.assertEqual(signature(nd), "(s)") - self.assertEqual(signature(TC.nd), "(s)") - self.assertEqual(signature(tc.nd), "()") - - def test_attribute_exception(self): - class NoCall: - def __getattr__(self, name): - raise BaseException - class CallA(NoCall): - def __call__(oui, a, b, c): - pass - class CallB(NoCall): - def __call__(self, ci): - pass - - for meth, mtip in ((NoCall, default_tip), (CallA, default_tip), - (NoCall(), ''), (CallA(), '(a, b, c)'), - (CallB(), '(ci)')): - self.assertEqual(signature(meth), mtip) - - def test_non_callables(self): - for obj in (0, 0.0, '0', b'0', [], {}): - self.assertEqual(signature(obj), '') - - -class Get_entityTest(unittest.TestCase): - def test_bad_entity(self): - self.assertIsNone(ct.get_entity('1/0')) - def test_good_entity(self): - self.assertIs(ct.get_entity('int'), int) - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_calltip_w.py python3.7-3.7.1/Lib/idlelib/idle_test/test_calltip_w.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_calltip_w.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_calltip_w.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,29 @@ +"Test calltip_w, coverage 18%." + +from idlelib import calltip_w +import unittest +from test.support import requires +from tkinter import Tk, Text + + +class CallTipWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.calltip = calltip_w.CalltipWindow(cls.text) + + @classmethod + def tearDownClass(cls): + cls.root.update_idletasks() + cls.root.destroy() + del cls.text, cls.root + + def test_init(self): + self.assertEqual(self.calltip.anchor_widget, self.text) + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_codecontext.py python3.7-3.7.1/Lib/idlelib/idle_test/test_codecontext.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_codecontext.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_codecontext.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,16 +1,12 @@ -"""Test idlelib.codecontext. - -Coverage: 100% -""" - -import re +"Test codecontext, coverage 100%" +from idlelib import codecontext import unittest -from unittest import mock from test.support import requires from tkinter import Tk, Frame, Text, TclError -import idlelib.codecontext as codecontext +from unittest import mock +import re from idlelib import config diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_colorizer.py python3.7-3.7.1/Lib/idlelib/idle_test/test_colorizer.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_colorizer.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_colorizer.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,10 +1,6 @@ -'''Test idlelib/colorizer.py +"Test colorizer, coverage 25%." -Perform minimal sanity checks that module imports and some things run. - -Coverage 22%. -''' -from idlelib import colorizer # always test import +from idlelib import colorizer from test.support import requires from tkinter import Tk, Text import unittest @@ -36,6 +32,7 @@ def test_colorizer(self): colorizer.color_config(self.text) + class ColorDelegatorTest(unittest.TestCase): @classmethod diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_configdialog.py python3.7-3.7.1/Lib/idlelib/idle_test/test_configdialog.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_configdialog.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_configdialog.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,7 +1,6 @@ -"""Test idlelib.configdialog. +"""Test configdialog, coverage 94%. Half the class creates dialog, half works with user customizations. -Coverage: 95%. """ from idlelib import configdialog from test.support import requires @@ -61,6 +60,7 @@ page = cls.page = dialog.fontpage dialog.note.select(page) page.set_samples = Func() # Mask instance method. + page.update() @classmethod def tearDownClass(cls): @@ -211,6 +211,7 @@ @classmethod def setUpClass(cls): cls.page = dialog.fontpage + cls.page.update() def test_load_tab_cfg(self): d = self.page @@ -241,6 +242,7 @@ page.paint_theme_sample = Func() page.set_highlight_target = Func() page.set_color_sample = Func() + page.update() @classmethod def tearDownClass(cls): @@ -1086,6 +1088,7 @@ dialog.note.select(page) page.set = page.set_add_delete_state = Func() page.upc = page.update_help_changes = Func() + page.update() @classmethod def tearDownClass(cls): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_config_key.py python3.7-3.7.1/Lib/idlelib/idle_test/test_config_key.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_config_key.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_config_key.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,7 +1,5 @@ -''' Test idlelib.config_key. +"Test config_key, coverage 75%" -Coverage: 56% from creating and closing dialog. -''' from idlelib import config_key from test.support import requires import sys diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_config.py python3.7-3.7.1/Lib/idlelib/idle_test/test_config.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_config.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_config.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,9 +1,9 @@ -'''Test idlelib.config. - -Coverage: 96% (100% for IdleConfParser, IdleUserConfParser*, ConfigChanges). +"""Test config, coverage 93%. +(100% for IdleConfParser, IdleUserConfParser*, ConfigChanges). * Exception is OSError clause in Save method. Much of IdleConf is also exercised by ConfigDialog and test_configdialog. -''' +""" +from idlelib import config import copy import sys import os @@ -12,7 +12,6 @@ import unittest from unittest import mock import idlelib -from idlelib import config from idlelib.idle_test.mock_idle import Func # Tests should not depend on fortuitous user configurations. @@ -256,9 +255,9 @@ with self.assertRaises(FileNotFoundError): conf.GetUserCfgDir() - @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for windows system') + @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for Windows system') def test_get_user_cfg_dir_windows(self): - "Test to get user config directory under windows" + "Test to get user config directory under Windows" conf = self.new_config(_utest=True) # Check normal way should success @@ -357,11 +356,11 @@ self.assertCountEqual( conf.GetSectionList('default', 'main'), - ['General', 'EditorWindow', 'Indent', 'Theme', + ['General', 'EditorWindow', 'PyShell', 'Indent', 'Theme', 'Keys', 'History', 'HelpFiles']) self.assertCountEqual( conf.GetSectionList('user', 'main'), - ['General', 'EditorWindow', 'Indent', 'Theme', + ['General', 'EditorWindow', 'PyShell', 'Indent', 'Theme', 'Keys', 'History', 'HelpFiles']) with self.assertRaises(config.InvalidConfigSet): @@ -453,7 +452,7 @@ self.assertCountEqual( conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')), - ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch','ZzDummy']) + ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', 'ZzDummy']) def test_get_extn_name_for_event(self): userextn.read_string(''' diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_debugger.py python3.7-3.7.1/Lib/idlelib/idle_test/test_debugger.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_debugger.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_debugger.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,11 +1,9 @@ -''' Test idlelib.debugger. +"Test debugger, coverage 19%" -Coverage: 19% -''' from idlelib import debugger +import unittest from test.support import requires requires('gui') -import unittest from tkinter import Tk @@ -25,5 +23,7 @@ debugger.NamespaceViewer(self.root, 'Test') +# Other classes are Idb, Debugger, and StackViewer. + if __name__ == '__main__': unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_debugger_r.py python3.7-3.7.1/Lib/idlelib/idle_test/test_debugger_r.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_debugger_r.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_debugger_r.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,29 @@ +"Test debugger_r, coverage 30%." + +from idlelib import debugger_r +import unittest +from test.support import requires +from tkinter import Tk + + +class Test(unittest.TestCase): + +## @classmethod +## def setUpClass(cls): +## requires('gui') +## cls.root = Tk() +## +## @classmethod +## def tearDownClass(cls): +## cls.root.destroy() +## del cls.root + + def test_init(self): + self.assertTrue(True) # Get coverage of import + + +# Classes GUIProxy, IdbAdapter, FrameProxy, CodeProxy, DictProxy, +# GUIAdapter, IdbProxy plus 7 module functions. + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_debugobj.py python3.7-3.7.1/Lib/idlelib/idle_test/test_debugobj.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_debugobj.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_debugobj.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,57 @@ +"Test debugobj, coverage 40%." + +from idlelib import debugobj +import unittest + + +class ObjectTreeItemTest(unittest.TestCase): + + def test_init(self): + ti = debugobj.ObjectTreeItem('label', 22) + self.assertEqual(ti.labeltext, 'label') + self.assertEqual(ti.object, 22) + self.assertEqual(ti.setfunction, None) + + +class ClassTreeItemTest(unittest.TestCase): + + def test_isexpandable(self): + ti = debugobj.ClassTreeItem('label', 0) + self.assertTrue(ti.IsExpandable()) + + +class AtomicObjectTreeItemTest(unittest.TestCase): + + def test_isexpandable(self): + ti = debugobj.AtomicObjectTreeItem('label', 0) + self.assertFalse(ti.IsExpandable()) + + +class SequenceTreeItemTest(unittest.TestCase): + + def test_isexpandable(self): + ti = debugobj.SequenceTreeItem('label', ()) + self.assertFalse(ti.IsExpandable()) + ti = debugobj.SequenceTreeItem('label', (1,)) + self.assertTrue(ti.IsExpandable()) + + def test_keys(self): + ti = debugobj.SequenceTreeItem('label', 'abc') + self.assertEqual(list(ti.keys()), [0, 1, 2]) + + +class DictTreeItemTest(unittest.TestCase): + + def test_isexpandable(self): + ti = debugobj.DictTreeItem('label', {}) + self.assertFalse(ti.IsExpandable()) + ti = debugobj.DictTreeItem('label', {1:1}) + self.assertTrue(ti.IsExpandable()) + + def test_keys(self): + ti = debugobj.DictTreeItem('label', {1:1, 0:0, 2:2}) + self.assertEqual(ti.keys(), [0, 1, 2]) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_debugobj_r.py python3.7-3.7.1/Lib/idlelib/idle_test/test_debugobj_r.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_debugobj_r.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_debugobj_r.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,22 @@ +"Test debugobj_r, coverage 56%." + +from idlelib import debugobj_r +import unittest + + +class WrappedObjectTreeItemTest(unittest.TestCase): + + def test_getattr(self): + ti = debugobj_r.WrappedObjectTreeItem(list) + self.assertEqual(ti.append, list.append) + +class StubObjectTreeItemTest(unittest.TestCase): + + def test_init(self): + ti = debugobj_r.StubObjectTreeItem('socket', 1111) + self.assertEqual(ti.sockio, 'socket') + self.assertEqual(ti.oid, 1111) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_delegator.py python3.7-3.7.1/Lib/idlelib/idle_test/test_delegator.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_delegator.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_delegator.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,5 +1,8 @@ -import unittest +"Test delegator, coverage 100%." + from idlelib.delegator import Delegator +import unittest + class DelegatorTest(unittest.TestCase): @@ -36,5 +39,6 @@ self.assertEqual(mydel._Delegator__cache, set()) self.assertIs(mydel.delegate, float) + if __name__ == '__main__': unittest.main(verbosity=2, exit=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_editor.py python3.7-3.7.1/Lib/idlelib/idle_test/test_editor.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_editor.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_editor.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,14 +1,46 @@ +"Test editor, coverage 35%." + +from idlelib import editor import unittest -from idlelib.editor import EditorWindow +from test.support import requires +from tkinter import Tk + +Editor = editor.EditorWindow + + +class EditorWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) + cls.root.destroy() + del cls.root + + def test_init(self): + e = Editor(root=self.root) + self.assertEqual(e.root, self.root) + e._close() + + +class EditorFunctionTest(unittest.TestCase): -class Editor_func_test(unittest.TestCase): def test_filename_to_unicode(self): - func = EditorWindow._filename_to_unicode - class dummy(): filesystemencoding = 'utf-8' + func = Editor._filename_to_unicode + class dummy(): + filesystemencoding = 'utf-8' pairs = (('abc', 'abc'), ('a\U00011111c', 'a\ufffdc'), (b'abc', 'abc'), (b'a\xf0\x91\x84\x91c', 'a\ufffdc')) for inp, out in pairs: self.assertEqual(func(dummy, inp), out) + if __name__ == '__main__': unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_filelist.py python3.7-3.7.1/Lib/idlelib/idle_test/test_filelist.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_filelist.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_filelist.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,33 @@ +"Test filelist, coverage 19%." + +from idlelib import filelist +import unittest +from test.support import requires +from tkinter import Tk + +class FileListTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) + cls.root.destroy() + del cls.root + + def test_new_empty(self): + flist = filelist.FileList(self.root) + self.assertEqual(flist.root, self.root) + e = flist.new() + self.assertEqual(type(e), flist.EditorWindow) + e._close() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_grep.py python3.7-3.7.1/Lib/idlelib/idle_test/test_grep.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_grep.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_grep.py 2018-10-20 06:04:19.000000000 +0000 @@ -3,14 +3,15 @@ dummy_command calls grep_it calls findfiles. An exception raised in one method will fail callers. Otherwise, tests are mostly independent. -*** Currently only test grep_it. +Currently only test grep_it, coverage 51%. """ +from idlelib.grep import GrepDialog import unittest from test.support import captured_stdout from idlelib.idle_test.mock_tk import Var -from idlelib.grep import GrepDialog import re + class Dummy_searchengine: '''GrepDialog.__init__ calls parent SearchDiabolBase which attaches the passed in SearchEngine instance as attribute 'engine'. Only a few of the @@ -21,6 +22,7 @@ searchengine = Dummy_searchengine() + class Dummy_grep: # Methods tested #default_command = GrepDialog.default_command @@ -34,6 +36,7 @@ grep = Dummy_grep() + class FindfilesTest(unittest.TestCase): # findfiles is really a function, not a method, could be iterator # test that filename return filename @@ -41,6 +44,7 @@ # test that recursive flag adds idle_test .py files pass + class Grep_itTest(unittest.TestCase): # Test captured reports with 0 and some hits. # Should test file names, but Windows reports have mixed / and \ separators @@ -71,10 +75,12 @@ self.assertIn('2', lines[3]) # hits found 2 self.assertTrue(lines[4].startswith('(Hint:')) + class Default_commandTest(unittest.TestCase): # To write this, move outwin import to top of GrepDialog # so it can be replaced by captured_stdout in class setup/teardown. pass + if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_help_about.py python3.7-3.7.1/Lib/idlelib/idle_test/test_help_about.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_help_about.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_help_about.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,18 +1,19 @@ -'''Test idlelib.help_about. +"""Test help_about, coverage 100%. +help_about.build_bits branches on sys.platform='darwin'. +'100% combines coverage on Mac and others. +""" -Coverage: 100% -''' +from idlelib import help_about +import unittest from test.support import requires, findfile from tkinter import Tk, TclError -import unittest -from unittest import mock from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func -from idlelib.help_about import AboutDialog as About -from idlelib import help_about from idlelib import textview import os.path -from platform import python_version, architecture +from platform import python_version + +About = help_about.AboutDialog class LiveDialogTest(unittest.TestCase): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_help.py python3.7-3.7.1/Lib/idlelib/idle_test/test_help.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_help.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_help.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,13 +1,12 @@ -'''Test idlelib.help. +"Test help, coverage 87%." -Coverage: 87% -''' from idlelib import help +import unittest from test.support import requires requires('gui') from os.path import abspath, dirname, join from tkinter import Tk -import unittest + class HelpFrameTest(unittest.TestCase): @@ -30,5 +29,6 @@ text = self.frame.text self.assertEqual(text.get('1.0', '1.end'), ' IDLE ') + if __name__ == '__main__': unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_history.py python3.7-3.7.1/Lib/idlelib/idle_test/test_history.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_history.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_history.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,15 +1,18 @@ +" Test history, coverage 100%." + +from idlelib.history import History import unittest from test.support import requires import tkinter as tk from tkinter import Text as tkText from idlelib.idle_test.mock_tk import Text as mkText -from idlelib.history import History from idlelib.config import idleConf line1 = 'a = 7' line2 = 'b = a' + class StoreTest(unittest.TestCase): '''Tests History.__init__ and History.store with mock Text''' @@ -61,6 +64,7 @@ def bell(self): self._bell = True + class FetchTest(unittest.TestCase): '''Test History.fetch with wrapped tk.Text. ''' diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_hyperparser.py python3.7-3.7.1/Lib/idlelib/idle_test/test_hyperparser.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_hyperparser.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_hyperparser.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,9 +1,10 @@ -"""Unittest for idlelib.hyperparser.py.""" +"Test hyperparser, coverage 98%." + +from idlelib.hyperparser import HyperParser import unittest from test.support import requires from tkinter import Tk, Text from idlelib.editor import EditorWindow -from idlelib.hyperparser import HyperParser class DummyEditwin: def __init__(self, text): @@ -270,5 +271,6 @@ self.assertEqual(eat_id('2' + 'a' * (length - 1), 0, length), 0) self.assertEqual(eat_id('2' + 'é' * (length - 1), 0, length), 0) + if __name__ == '__main__': unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_iomenu.py python3.7-3.7.1/Lib/idlelib/idle_test/test_iomenu.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_iomenu.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_iomenu.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,233 +1,36 @@ +"Test , coverage 16%." + +from idlelib import iomenu import unittest -import io +from test.support import requires +from tkinter import Tk + +from idlelib.editor import EditorWindow -from idlelib.run import PseudoInputFile, PseudoOutputFile +class IOBindigTest(unittest.TestCase): -class S(str): - def __str__(self): - return '%s:str' % type(self).__name__ - def __unicode__(self): - return '%s:unicode' % type(self).__name__ - def __len__(self): - return 3 - def __iter__(self): - return iter('abc') - def __getitem__(self, *args): - return '%s:item' % type(self).__name__ - def __getslice__(self, *args): - return '%s:slice' % type(self).__name__ - -class MockShell: - def __init__(self): - self.reset() - - def write(self, *args): - self.written.append(args) - - def readline(self): - return self.lines.pop() - - def close(self): - pass - - def reset(self): - self.written = [] - - def push(self, lines): - self.lines = list(lines)[::-1] - - -class PseudeOutputFilesTest(unittest.TestCase): - def test_misc(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) - self.assertIsNone(f.newlines) - self.assertEqual(f.name, '') - self.assertFalse(f.closed) - self.assertTrue(f.isatty()) - self.assertFalse(f.readable()) - self.assertTrue(f.writable()) - self.assertFalse(f.seekable()) - - def test_unsupported(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertRaises(OSError, f.fileno) - self.assertRaises(OSError, f.tell) - self.assertRaises(OSError, f.seek, 0) - self.assertRaises(OSError, f.read, 0) - self.assertRaises(OSError, f.readline, 0) - - def test_write(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - f.write('test') - self.assertEqual(shell.written, [('test', 'stdout')]) - shell.reset() - f.write('t\xe8st') - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - shell.reset() - - f.write(S('t\xe8st')) - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - self.assertEqual(type(shell.written[0][0]), str) - shell.reset() - - self.assertRaises(TypeError, f.write) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.write, b'test') - self.assertRaises(TypeError, f.write, 123) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.write, 'test', 'spam') - self.assertEqual(shell.written, []) - - def test_writelines(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - f.writelines([]) - self.assertEqual(shell.written, []) - shell.reset() - f.writelines(['one\n', 'two']) - self.assertEqual(shell.written, - [('one\n', 'stdout'), ('two', 'stdout')]) - shell.reset() - f.writelines(['on\xe8\n', 'tw\xf2']) - self.assertEqual(shell.written, - [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')]) - shell.reset() - - f.writelines([S('t\xe8st')]) - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - self.assertEqual(type(shell.written[0][0]), str) - shell.reset() - - self.assertRaises(TypeError, f.writelines) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, 123) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, [b'test']) - self.assertRaises(TypeError, f.writelines, [123]) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, [], []) - self.assertEqual(shell.written, []) - - def test_close(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertFalse(f.closed) - f.write('test') - f.close() - self.assertTrue(f.closed) - self.assertRaises(ValueError, f.write, 'x') - self.assertEqual(shell.written, [('test', 'stdout')]) - f.close() - self.assertRaises(TypeError, f.close, 1) - - -class PseudeInputFilesTest(unittest.TestCase): - def test_misc(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) - self.assertIsNone(f.newlines) - self.assertEqual(f.name, '') - self.assertFalse(f.closed) - self.assertTrue(f.isatty()) - self.assertTrue(f.readable()) - self.assertFalse(f.writable()) - self.assertFalse(f.seekable()) - - def test_unsupported(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - self.assertRaises(OSError, f.fileno) - self.assertRaises(OSError, f.tell) - self.assertRaises(OSError, f.seek, 0) - self.assertRaises(OSError, f.write, 'x') - self.assertRaises(OSError, f.writelines, ['x']) - - def test_read(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(), 'one\ntwo\n') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(-1), 'one\ntwo\n') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(None), 'one\ntwo\n') - shell.push(['one\n', 'two\n', 'three\n', '']) - self.assertEqual(f.read(2), 'on') - self.assertEqual(f.read(3), 'e\nt') - self.assertEqual(f.read(10), 'wo\nthree\n') - - shell.push(['one\n', 'two\n']) - self.assertEqual(f.read(0), '') - self.assertRaises(TypeError, f.read, 1.5) - self.assertRaises(TypeError, f.read, '1') - self.assertRaises(TypeError, f.read, 1, 1) - - def test_readline(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', 'three\n', 'four\n']) - self.assertEqual(f.readline(), 'one\n') - self.assertEqual(f.readline(-1), 'two\n') - self.assertEqual(f.readline(None), 'three\n') - shell.push(['one\ntwo\n']) - self.assertEqual(f.readline(), 'one\n') - self.assertEqual(f.readline(), 'two\n') - shell.push(['one', 'two', 'three']) - self.assertEqual(f.readline(), 'one') - self.assertEqual(f.readline(), 'two') - shell.push(['one\n', 'two\n', 'three\n']) - self.assertEqual(f.readline(2), 'on') - self.assertEqual(f.readline(1), 'e') - self.assertEqual(f.readline(1), '\n') - self.assertEqual(f.readline(10), 'two\n') - - shell.push(['one\n', 'two\n']) - self.assertEqual(f.readline(0), '') - self.assertRaises(TypeError, f.readlines, 1.5) - self.assertRaises(TypeError, f.readlines, '1') - self.assertRaises(TypeError, f.readlines, 1, 1) - - def test_readlines(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(-1), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(None), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(0), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(3), ['one\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(4), ['one\n', 'two\n']) - - shell.push(['one\n', 'two\n', '']) - self.assertRaises(TypeError, f.readlines, 1.5) - self.assertRaises(TypeError, f.readlines, '1') - self.assertRaises(TypeError, f.readlines, 1, 1) - - def test_close(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertFalse(f.closed) - self.assertEqual(f.readline(), 'one\n') - f.close() - self.assertFalse(f.closed) - self.assertEqual(f.readline(), 'two\n') - self.assertRaises(TypeError, f.close, 1) + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.editwin = EditorWindow(root=cls.root) + + @classmethod + def tearDownClass(cls): + cls.editwin._close() + del cls.editwin + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_init(self): + io = iomenu.IOBinding(self.editwin) + self.assertIs(io.editwin, self.editwin) + io.close if __name__ == '__main__': diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_macosx.py python3.7-3.7.1/Lib/idlelib/idle_test/test_macosx.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_macosx.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_macosx.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,11 +1,9 @@ -'''Test idlelib.macosx.py. +"Test macosx, coverage 45% on Windows." -Coverage: 71% on Windows. -''' from idlelib import macosx +import unittest from test.support import requires import tkinter as tk -import unittest import unittest.mock as mock from idlelib.filelist import FileList diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_mainmenu.py python3.7-3.7.1/Lib/idlelib/idle_test/test_mainmenu.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_mainmenu.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_mainmenu.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,21 @@ +"Test mainmenu, coverage 100%." +# Reported as 88%; mocking turtledemo absence would have no point. + +from idlelib import mainmenu +import unittest + + +class MainMenuTest(unittest.TestCase): + + def test_menudefs(self): + actual = [item[0] for item in mainmenu.menudefs] + expect = ['file', 'edit', 'format', 'run', 'shell', + 'debug', 'options', 'window', 'help'] + self.assertEqual(actual, expect) + + def test_default_keydefs(self): + self.assertGreaterEqual(len(mainmenu.default_keydefs), 50) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_multicall.py python3.7-3.7.1/Lib/idlelib/idle_test/test_multicall.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_multicall.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_multicall.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,40 @@ +"Test multicall, coverage 33%." + +from idlelib import multicall +import unittest +from test.support import requires +from tkinter import Tk, Text + + +class MultiCallTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.mc = multicall.MultiCallCreator(Text) + + @classmethod + def tearDownClass(cls): + del cls.mc + cls.root.update_idletasks() +## for id in cls.root.tk.call('after', 'info'): +## cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_creator(self): + mc = self.mc + self.assertIs(multicall._multicall_dict[Text], mc) + self.assertTrue(issubclass(mc, Text)) + mc2 = multicall.MultiCallCreator(Text) + self.assertIs(mc, mc2) + + def test_init(self): + mctext = self.mc(self.root) + self.assertIsInstance(mctext._MultiCall__binders, list) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_outwin.py python3.7-3.7.1/Lib/idlelib/idle_test/test_outwin.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_outwin.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_outwin.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,12 +1,11 @@ -""" Test idlelib.outwin. -""" +"Test outwin, coverage 76%." +from idlelib import outwin import unittest +from test.support import requires from tkinter import Tk, Text from idlelib.idle_test.mock_tk import Mbox_func from idlelib.idle_test.mock_idle import Func -from idlelib import outwin -from test.support import requires from unittest import mock diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_paragraph.py python3.7-3.7.1/Lib/idlelib/idle_test/test_paragraph.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_paragraph.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_paragraph.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,9 +1,10 @@ -# Test the functions and main class method of paragraph.py +"Test paragraph, coverage 76%." + +from idlelib import paragraph as pg import unittest -from idlelib import paragraph as fp -from idlelib.editor import EditorWindow -from tkinter import Tk, Text from test.support import requires +from tkinter import Tk, Text +from idlelib.editor import EditorWindow class Is_Get_Test(unittest.TestCase): @@ -15,26 +16,26 @@ leadingws_nocomment = ' This is not a comment' def test_is_all_white(self): - self.assertTrue(fp.is_all_white('')) - self.assertTrue(fp.is_all_white('\t\n\r\f\v')) - self.assertFalse(fp.is_all_white(self.test_comment)) + self.assertTrue(pg.is_all_white('')) + self.assertTrue(pg.is_all_white('\t\n\r\f\v')) + self.assertFalse(pg.is_all_white(self.test_comment)) def test_get_indent(self): Equal = self.assertEqual - Equal(fp.get_indent(self.test_comment), '') - Equal(fp.get_indent(self.trailingws_comment), '') - Equal(fp.get_indent(self.leadingws_comment), ' ') - Equal(fp.get_indent(self.leadingws_nocomment), ' ') + Equal(pg.get_indent(self.test_comment), '') + Equal(pg.get_indent(self.trailingws_comment), '') + Equal(pg.get_indent(self.leadingws_comment), ' ') + Equal(pg.get_indent(self.leadingws_nocomment), ' ') def test_get_comment_header(self): Equal = self.assertEqual # Test comment strings - Equal(fp.get_comment_header(self.test_comment), '#') - Equal(fp.get_comment_header(self.trailingws_comment), '#') - Equal(fp.get_comment_header(self.leadingws_comment), ' #') + Equal(pg.get_comment_header(self.test_comment), '#') + Equal(pg.get_comment_header(self.trailingws_comment), '#') + Equal(pg.get_comment_header(self.leadingws_comment), ' #') # Test non-comment strings - Equal(fp.get_comment_header(self.leadingws_nocomment), ' ') - Equal(fp.get_comment_header(self.test_nocomment), '') + Equal(pg.get_comment_header(self.leadingws_nocomment), ' ') + Equal(pg.get_comment_header(self.test_nocomment), '') class FindTest(unittest.TestCase): @@ -62,7 +63,7 @@ linelength = int(text.index("%d.end" % line).split('.')[1]) for col in (0, linelength//2, linelength): tempindex = "%d.%d" % (line, col) - self.assertEqual(fp.find_paragraph(text, tempindex), expected) + self.assertEqual(pg.find_paragraph(text, tempindex), expected) text.delete('1.0', 'end') def test_find_comment(self): @@ -161,7 +162,7 @@ def test_reformat_paragraph(self): Equal = self.assertEqual - reform = fp.reformat_paragraph + reform = pg.reformat_paragraph hw = "O hello world" Equal(reform(' ', 1), ' ') Equal(reform("Hello world", 20), "Hello world") @@ -192,7 +193,7 @@ test_string = ( " \"\"\"this is a test of a reformat for a triple quoted string" " will it reformat to less than 70 characters for me?\"\"\"") - result = fp.reformat_comment(test_string, 70, " ") + result = pg.reformat_comment(test_string, 70, " ") expected = ( " \"\"\"this is a test of a reformat for a triple quoted string will it\n" " reformat to less than 70 characters for me?\"\"\"") @@ -201,7 +202,7 @@ test_comment = ( "# this is a test of a reformat for a triple quoted string will " "it reformat to less than 70 characters for me?") - result = fp.reformat_comment(test_comment, 70, "#") + result = pg.reformat_comment(test_comment, 70, "#") expected = ( "# this is a test of a reformat for a triple quoted string will it\n" "# reformat to less than 70 characters for me?") @@ -210,7 +211,7 @@ class FormatClassTest(unittest.TestCase): def test_init_close(self): - instance = fp.FormatParagraph('editor') + instance = pg.FormatParagraph('editor') self.assertEqual(instance.editwin, 'editor') instance.close() self.assertEqual(instance.editwin, None) @@ -269,14 +270,16 @@ def setUpClass(cls): requires('gui') cls.root = Tk() + cls.root.withdraw() editor = Editor(root=cls.root) cls.text = editor.text.text # Test code does not need the wrapper. - cls.formatter = fp.FormatParagraph(editor).format_paragraph_event + cls.formatter = pg.FormatParagraph(editor).format_paragraph_event # Sets the insert mark just after the re-wrapped and inserted text. @classmethod def tearDownClass(cls): del cls.text, cls.formatter + cls.root.update_idletasks() cls.root.destroy() del cls.root diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_parenmatch.py python3.7-3.7.1/Lib/idlelib/idle_test/test_parenmatch.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_parenmatch.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_parenmatch.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,8 +1,8 @@ -'''Test idlelib.parenmatch. +"""Test parenmatch, coverage 91%. This must currently be a gui test because ParenMatch methods use several text methods not defined on idlelib.idle_test.mock_tk.Text. -''' +""" from idlelib.parenmatch import ParenMatch from test.support import requires requires('gui') diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_pathbrowser.py python3.7-3.7.1/Lib/idlelib/idle_test/test_pathbrowser.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_pathbrowser.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_pathbrowser.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,19 +1,17 @@ -""" Test idlelib.pathbrowser. -""" +"Test pathbrowser, coverage 95%." +from idlelib import pathbrowser +import unittest +from test.support import requires +from tkinter import Tk import os.path import pyclbr # for _modules import sys # for sys.path -from tkinter import Tk -from test.support import requires -import unittest from idlelib.idle_test.mock_idle import Func - import idlelib # for __file__ from idlelib import browser -from idlelib import pathbrowser from idlelib.tree import TreeNode diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_percolator.py python3.7-3.7.1/Lib/idlelib/idle_test/test_percolator.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_percolator.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_percolator.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,10 +1,10 @@ -'''Test percolator.py.''' -from test.support import requires -requires('gui') +"Test percolator, coverage 100%." +from idlelib.percolator import Percolator, Delegator import unittest +from test.support import requires +requires('gui') from tkinter import Text, Tk, END -from idlelib.percolator import Percolator, Delegator class MyFilter(Delegator): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_pyparse.py python3.7-3.7.1/Lib/idlelib/idle_test/test_pyparse.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_pyparse.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_pyparse.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,11 +1,8 @@ -"""Unittest for idlelib.pyparse.py. +"Test pyparse, coverage 96%." -Coverage: 97% -""" - -from collections import namedtuple -import unittest from idlelib import pyparse +import unittest +from collections import namedtuple class ParseMapTest(unittest.TestCase): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_pyshell.py python3.7-3.7.1/Lib/idlelib/idle_test/test_pyshell.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_pyshell.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_pyshell.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,42 @@ +"Test pyshell, coverage 12%." +# Plus coverage of test_warning. Was 20% with test_openshell. + +from idlelib import pyshell +import unittest +from test.support import requires +from tkinter import Tk + + +class PyShellFileListTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + #cls.root.update_idletasks() +## for id in cls.root.tk.call('after', 'info'): +## cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_init(self): + psfl = pyshell.PyShellFileList(self.root) + self.assertEqual(psfl.EditorWindow, pyshell.PyShellEditorWindow) + self.assertIsNone(psfl.pyshell) + +# The following sometimes causes 'invalid command name "109734456recolorize"'. +# Uncommenting after_cancel above prevents this, but results in +# TclError: bad window path name ".!listedtoplevel.!frame.text" +# which is normally prevented by after_cancel. +## def test_openshell(self): +## pyshell.use_subprocess = False +## ps = pyshell.PyShellFileList(self.root).open_shell() +## self.assertIsInstance(ps, pyshell.PyShell) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_query.py python3.7-3.7.1/Lib/idlelib/idle_test/test_query.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_query.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_query.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,4 +1,4 @@ -"""Test idlelib.query. +"""Test query, coverage 91%). Non-gui tests for Query, SectionName, ModuleName, and HelpSource use dummy versions that extract the non-gui methods and add other needed @@ -8,17 +8,15 @@ The appearance of the widgets is checked by the Query and HelpSource htests. These are run by running query.py. - -Coverage: 94% (100% for Query and SectionName). -6 of 8 missing are ModuleName exceptions I don't know how to trigger. """ +from idlelib import query +import unittest from test.support import requires -import sys from tkinter import Tk -import unittest + +import sys from unittest import mock from idlelib.idle_test.mock_tk import Var -from idlelib import query # NON-GUI TESTS diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_redirector.py python3.7-3.7.1/Lib/idlelib/idle_test/test_redirector.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_redirector.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_redirector.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,12 +1,10 @@ -'''Test idlelib.redirector. +"Test redirector, coverage 100%." -100% coverage -''' -from test.support import requires +from idlelib.redirector import WidgetRedirector import unittest -from idlelib.idle_test.mock_idle import Func +from test.support import requires from tkinter import Tk, Text, TclError -from idlelib.redirector import WidgetRedirector +from idlelib.idle_test.mock_idle import Func class InitCloseTest(unittest.TestCase): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_replace.py python3.7-3.7.1/Lib/idlelib/idle_test/test_replace.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_replace.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_replace.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,13 +1,14 @@ -"""Unittest for idlelib.replace.py""" +"Test replace, coverage 78%." + +from idlelib.replace import ReplaceDialog +import unittest from test.support import requires requires('gui') +from tkinter import Tk, Text -import unittest from unittest.mock import Mock -from tkinter import Tk, Text from idlelib.idle_test.mock_tk import Mbox import idlelib.searchengine as se -from idlelib.replace import ReplaceDialog orig_mbox = se.tkMessageBox showerror = Mbox.showerror diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_rpc.py python3.7-3.7.1/Lib/idlelib/idle_test/test_rpc.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_rpc.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_rpc.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,30 @@ +"Test rpc, coverage 20%." + +from idlelib import rpc +import unittest + +import marshal + + +class CodePicklerTest(unittest.TestCase): + + def test_pickle_unpickle(self): + def f(): return a + b + c + func, (cbytes,) = rpc.pickle_code(f.__code__) + self.assertIs(func, rpc.unpickle_code) + self.assertIn(b'test_rpc.py', cbytes) + code = rpc.unpickle_code(cbytes) + self.assertEqual(code.co_names, ('a', 'b', 'c')) + + def test_code_pickler(self): + self.assertIn(type((lambda:None).__code__), + rpc.CodePickler.dispatch_table) + + def test_dumps(self): + def f(): pass + # The main test here is that pickling code does not raise. + self.assertIn(b'test_rpc.py', rpc.dumps(f.__code__)) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_rstrip.py python3.7-3.7.1/Lib/idlelib/idle_test/test_rstrip.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_rstrip.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_rstrip.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,5 +1,7 @@ +"Test rstrip, coverage 100%." + +from idlelib import rstrip import unittest -import idlelib.rstrip as rs from idlelib.idle_test.mock_idle import Editor class rstripTest(unittest.TestCase): @@ -7,7 +9,7 @@ def test_rstrip_line(self): editor = Editor() text = editor.text - do_rstrip = rs.RstripExtension(editor).do_rstrip + do_rstrip = rstrip.Rstrip(editor).do_rstrip do_rstrip() self.assertEqual(text.get('1.0', 'insert'), '') @@ -20,12 +22,12 @@ def test_rstrip_multiple(self): editor = Editor() - # Uncomment following to verify that test passes with real widgets. -## from idlelib.editor import EditorWindow as Editor -## from tkinter import Tk -## editor = Editor(root=Tk()) + # Comment above, uncomment 3 below to test with real Editor & Text. + #from idlelib.editor import EditorWindow as Editor + #from tkinter import Tk + #editor = Editor(root=Tk()) text = editor.text - do_rstrip = rs.RstripExtension(editor).do_rstrip + do_rstrip = rstrip.Rstrip(editor).do_rstrip original = ( "Line with an ending tab \n" @@ -45,5 +47,7 @@ do_rstrip() self.assertEqual(text.get('1.0', 'insert'), stripped) + + if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_run.py python3.7-3.7.1/Lib/idlelib/idle_test/test_run.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_run.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_run.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,11 +1,14 @@ +"Test run, coverage 42%." + +from idlelib import run import unittest from unittest import mock - from test.support import captured_stderr -import idlelib.run as idlerun +import io class RunTest(unittest.TestCase): + def test_print_exception_unhashable(self): class UnhashableException(Exception): def __eq__(self, other): @@ -20,10 +23,10 @@ raise ex1 except UnhashableException: with captured_stderr() as output: - with mock.patch.object(idlerun, + with mock.patch.object(run, 'cleanup_traceback') as ct: ct.side_effect = lambda t, e: t - idlerun.print_exception() + run.print_exception() tb = output.getvalue().strip().splitlines() self.assertEqual(11, len(tb)) @@ -31,5 +34,231 @@ self.assertIn('UnhashableException: ex1', tb[10]) +# PseudoFile tests. + +class S(str): + def __str__(self): + return '%s:str' % type(self).__name__ + def __unicode__(self): + return '%s:unicode' % type(self).__name__ + def __len__(self): + return 3 + def __iter__(self): + return iter('abc') + def __getitem__(self, *args): + return '%s:item' % type(self).__name__ + def __getslice__(self, *args): + return '%s:slice' % type(self).__name__ + + +class MockShell: + def __init__(self): + self.reset() + def write(self, *args): + self.written.append(args) + def readline(self): + return self.lines.pop() + def close(self): + pass + def reset(self): + self.written = [] + def push(self, lines): + self.lines = list(lines)[::-1] + + +class PseudeInputFilesTest(unittest.TestCase): + + def test_misc(self): + shell = MockShell() + f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.encoding, 'utf-8') + self.assertIsNone(f.errors) + self.assertIsNone(f.newlines) + self.assertEqual(f.name, '') + self.assertFalse(f.closed) + self.assertTrue(f.isatty()) + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertFalse(f.seekable()) + + def test_unsupported(self): + shell = MockShell() + f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + self.assertRaises(OSError, f.fileno) + self.assertRaises(OSError, f.tell) + self.assertRaises(OSError, f.seek, 0) + self.assertRaises(OSError, f.write, 'x') + self.assertRaises(OSError, f.writelines, ['x']) + + def test_read(self): + shell = MockShell() + f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(), 'one\ntwo\n') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(-1), 'one\ntwo\n') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(None), 'one\ntwo\n') + shell.push(['one\n', 'two\n', 'three\n', '']) + self.assertEqual(f.read(2), 'on') + self.assertEqual(f.read(3), 'e\nt') + self.assertEqual(f.read(10), 'wo\nthree\n') + + shell.push(['one\n', 'two\n']) + self.assertEqual(f.read(0), '') + self.assertRaises(TypeError, f.read, 1.5) + self.assertRaises(TypeError, f.read, '1') + self.assertRaises(TypeError, f.read, 1, 1) + + def test_readline(self): + shell = MockShell() + f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', 'three\n', 'four\n']) + self.assertEqual(f.readline(), 'one\n') + self.assertEqual(f.readline(-1), 'two\n') + self.assertEqual(f.readline(None), 'three\n') + shell.push(['one\ntwo\n']) + self.assertEqual(f.readline(), 'one\n') + self.assertEqual(f.readline(), 'two\n') + shell.push(['one', 'two', 'three']) + self.assertEqual(f.readline(), 'one') + self.assertEqual(f.readline(), 'two') + shell.push(['one\n', 'two\n', 'three\n']) + self.assertEqual(f.readline(2), 'on') + self.assertEqual(f.readline(1), 'e') + self.assertEqual(f.readline(1), '\n') + self.assertEqual(f.readline(10), 'two\n') + + shell.push(['one\n', 'two\n']) + self.assertEqual(f.readline(0), '') + self.assertRaises(TypeError, f.readlines, 1.5) + self.assertRaises(TypeError, f.readlines, '1') + self.assertRaises(TypeError, f.readlines, 1, 1) + + def test_readlines(self): + shell = MockShell() + f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(-1), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(None), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(0), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(3), ['one\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(4), ['one\n', 'two\n']) + + shell.push(['one\n', 'two\n', '']) + self.assertRaises(TypeError, f.readlines, 1.5) + self.assertRaises(TypeError, f.readlines, '1') + self.assertRaises(TypeError, f.readlines, 1, 1) + + def test_close(self): + shell = MockShell() + f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertFalse(f.closed) + self.assertEqual(f.readline(), 'one\n') + f.close() + self.assertFalse(f.closed) + self.assertEqual(f.readline(), 'two\n') + self.assertRaises(TypeError, f.close, 1) + + +class PseudeOutputFilesTest(unittest.TestCase): + + def test_misc(self): + shell = MockShell() + f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.encoding, 'utf-8') + self.assertIsNone(f.errors) + self.assertIsNone(f.newlines) + self.assertEqual(f.name, '') + self.assertFalse(f.closed) + self.assertTrue(f.isatty()) + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertFalse(f.seekable()) + + def test_unsupported(self): + shell = MockShell() + f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertRaises(OSError, f.fileno) + self.assertRaises(OSError, f.tell) + self.assertRaises(OSError, f.seek, 0) + self.assertRaises(OSError, f.read, 0) + self.assertRaises(OSError, f.readline, 0) + + def test_write(self): + shell = MockShell() + f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + f.write('test') + self.assertEqual(shell.written, [('test', 'stdout')]) + shell.reset() + f.write('t\xe8st') + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + shell.reset() + + f.write(S('t\xe8st')) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + + self.assertRaises(TypeError, f.write) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, b'test') + self.assertRaises(TypeError, f.write, 123) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, 'test', 'spam') + self.assertEqual(shell.written, []) + + def test_writelines(self): + shell = MockShell() + f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + f.writelines([]) + self.assertEqual(shell.written, []) + shell.reset() + f.writelines(['one\n', 'two']) + self.assertEqual(shell.written, + [('one\n', 'stdout'), ('two', 'stdout')]) + shell.reset() + f.writelines(['on\xe8\n', 'tw\xf2']) + self.assertEqual(shell.written, + [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')]) + shell.reset() + + f.writelines([S('t\xe8st')]) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + + self.assertRaises(TypeError, f.writelines) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, 123) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, [b'test']) + self.assertRaises(TypeError, f.writelines, [123]) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, [], []) + self.assertEqual(shell.written, []) + + def test_close(self): + shell = MockShell() + f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertFalse(f.closed) + f.write('test') + f.close() + self.assertTrue(f.closed) + self.assertRaises(ValueError, f.write, 'x') + self.assertEqual(shell.written, [('test', 'stdout')]) + f.close() + self.assertRaises(TypeError, f.close, 1) + + if __name__ == '__main__': unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_runscript.py python3.7-3.7.1/Lib/idlelib/idle_test/test_runscript.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_runscript.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_runscript.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,33 @@ +"Test runscript, coverage 16%." + +from idlelib import runscript +import unittest +from test.support import requires +from tkinter import Tk +from idlelib.editor import EditorWindow + + +class ScriptBindingTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_init(self): + ew = EditorWindow(root=self.root) + sb = runscript.ScriptBinding(ew) + ew._close() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_scrolledlist.py python3.7-3.7.1/Lib/idlelib/idle_test/test_scrolledlist.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_scrolledlist.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_scrolledlist.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,11 +1,9 @@ -''' Test idlelib.scrolledlist. +"Test scrolledlist, coverage 38%." -Coverage: 39% -''' -from idlelib import scrolledlist +from idlelib.scrolledlist import ScrolledList +import unittest from test.support import requires requires('gui') -import unittest from tkinter import Tk @@ -22,7 +20,7 @@ def test_init(self): - scrolledlist.ScrolledList(self.root) + ScrolledList(self.root) if __name__ == '__main__': diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_searchbase.py python3.7-3.7.1/Lib/idlelib/idle_test/test_searchbase.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_searchbase.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_searchbase.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,8 +1,7 @@ -'''tests idlelib.searchbase. +"Test searchbase, coverage 98%." +# The only thing not covered is inconsequential -- +# testing skipping of suite when self.needwrapbutton is false. -Coverage: 99%. The only thing not covered is inconsequential -- -testing skipping of suite when self.needwrapbutton is false. -''' import unittest from test.support import requires from tkinter import Tk, Frame ##, BooleanVar, StringVar @@ -22,6 +21,7 @@ ## se.BooleanVar = BooleanVar ## se.StringVar = StringVar + class SearchDialogBaseTest(unittest.TestCase): @classmethod diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_searchengine.py python3.7-3.7.1/Lib/idlelib/idle_test/test_searchengine.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_searchengine.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_searchengine.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,18 +1,19 @@ -'''Test functions and SearchEngine class in idlelib.searchengine.py.''' +"Test searchengine, coverage 99%." -# With mock replacements, the module does not use any gui widgets. -# The use of tk.Text is avoided (for now, until mock Text is improved) -# by patching instances with an index function returning what is needed. -# This works because mock Text.get does not use .index. - -import re +from idlelib import searchengine as se import unittest # from test.support import requires from tkinter import BooleanVar, StringVar, TclError # ,Tk, Text import tkinter.messagebox as tkMessageBox -from idlelib import searchengine as se from idlelib.idle_test.mock_tk import Var, Mbox from idlelib.idle_test.mock_tk import Text as mockText +import re + +# With mock replacements, the module does not use any gui widgets. +# The use of tk.Text is avoided (for now, until mock Text is improved) +# by patching instances with an index function returning what is needed. +# This works because mock Text.get does not use .index. +# The tkinter imports are used to restore searchengine. def setUpModule(): # Replace s-e module tkinter imports other than non-gui TclError. @@ -326,4 +327,4 @@ if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_search.py python3.7-3.7.1/Lib/idlelib/idle_test/test_search.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_search.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_search.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,25 +1,23 @@ -"""Test SearchDialog class in idlelib.search.py""" +"Test search, coverage 69%." + +from idlelib import search +import unittest +from test.support import requires +requires('gui') +from tkinter import Tk, Text, BooleanVar +from idlelib import searchengine # Does not currently test the event handler wrappers. # A usage test should simulate clicks and check highlighting. # Tests need to be coordinated with SearchDialogBase tests # to avoid duplication. -from test.support import requires -requires('gui') - -import unittest -import tkinter as tk -from tkinter import BooleanVar -import idlelib.searchengine as se -import idlelib.search as sd - class SearchDialogTest(unittest.TestCase): @classmethod def setUpClass(cls): - cls.root = tk.Tk() + cls.root = Tk() @classmethod def tearDownClass(cls): @@ -27,10 +25,10 @@ del cls.root def setUp(self): - self.engine = se.SearchEngine(self.root) - self.dialog = sd.SearchDialog(self.root, self.engine) + self.engine = searchengine.SearchEngine(self.root) + self.dialog = search.SearchDialog(self.root, self.engine) self.dialog.bell = lambda: None - self.text = tk.Text(self.root) + self.text = Text(self.root) self.text.insert('1.0', 'Hello World!') def test_find_again(self): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_squeezer.py python3.7-3.7.1/Lib/idlelib/idle_test/test_squeezer.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_squeezer.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_squeezer.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,509 @@ +from collections import namedtuple +from tkinter import Text, Tk +import unittest +from unittest.mock import Mock, NonCallableMagicMock, patch, sentinel, ANY +from test.support import requires + +from idlelib.config import idleConf +from idlelib.squeezer import count_lines_with_wrapping, ExpandingButton, \ + Squeezer +from idlelib import macosx +from idlelib.textview import view_text +from idlelib.tooltip import Hovertip +from idlelib.pyshell import PyShell + + +SENTINEL_VALUE = sentinel.SENTINEL_VALUE + + +def get_test_tk_root(test_instance): + """Helper for tests: Create a root Tk object.""" + requires('gui') + root = Tk() + root.withdraw() + + def cleanup_root(): + root.update_idletasks() + root.destroy() + test_instance.addCleanup(cleanup_root) + + return root + + +class CountLinesTest(unittest.TestCase): + """Tests for the count_lines_with_wrapping function.""" + def check(self, expected, text, linewidth, tabwidth): + return self.assertEqual( + expected, + count_lines_with_wrapping(text, linewidth, tabwidth), + ) + + def test_count_empty(self): + """Test with an empty string.""" + self.assertEqual(count_lines_with_wrapping(""), 0) + + def test_count_begins_with_empty_line(self): + """Test with a string which begins with a newline.""" + self.assertEqual(count_lines_with_wrapping("\ntext"), 2) + + def test_count_ends_with_empty_line(self): + """Test with a string which ends with a newline.""" + self.assertEqual(count_lines_with_wrapping("text\n"), 1) + + def test_count_several_lines(self): + """Test with several lines of text.""" + self.assertEqual(count_lines_with_wrapping("1\n2\n3\n"), 3) + + def test_tab_width(self): + """Test with various tab widths and line widths.""" + self.check(expected=1, text='\t' * 1, linewidth=8, tabwidth=4) + self.check(expected=1, text='\t' * 2, linewidth=8, tabwidth=4) + self.check(expected=2, text='\t' * 3, linewidth=8, tabwidth=4) + self.check(expected=2, text='\t' * 4, linewidth=8, tabwidth=4) + self.check(expected=3, text='\t' * 5, linewidth=8, tabwidth=4) + + # test longer lines and various tab widths + self.check(expected=4, text='\t' * 10, linewidth=12, tabwidth=4) + self.check(expected=10, text='\t' * 10, linewidth=12, tabwidth=8) + self.check(expected=2, text='\t' * 4, linewidth=10, tabwidth=3) + + # test tabwidth=1 + self.check(expected=2, text='\t' * 9, linewidth=5, tabwidth=1) + self.check(expected=2, text='\t' * 10, linewidth=5, tabwidth=1) + self.check(expected=3, text='\t' * 11, linewidth=5, tabwidth=1) + + # test for off-by-one errors + self.check(expected=2, text='\t' * 6, linewidth=12, tabwidth=4) + self.check(expected=3, text='\t' * 6, linewidth=11, tabwidth=4) + self.check(expected=2, text='\t' * 6, linewidth=13, tabwidth=4) + + +class SqueezerTest(unittest.TestCase): + """Tests for the Squeezer class.""" + def make_mock_editor_window(self): + """Create a mock EditorWindow instance.""" + editwin = NonCallableMagicMock() + # isinstance(editwin, PyShell) must be true for Squeezer to enable + # auto-squeezing; in practice this will always be true + editwin.__class__ = PyShell + return editwin + + def make_squeezer_instance(self, editor_window=None): + """Create an actual Squeezer instance with a mock EditorWindow.""" + if editor_window is None: + editor_window = self.make_mock_editor_window() + return Squeezer(editor_window) + + def test_count_lines(self): + """Test Squeezer.count_lines() with various inputs. + + This checks that Squeezer.count_lines() calls the + count_lines_with_wrapping() function with the appropriate parameters. + """ + for tabwidth, linewidth in [(4, 80), (1, 79), (8, 80), (3, 120)]: + self._test_count_lines_helper(linewidth=linewidth, + tabwidth=tabwidth) + + def _prepare_mock_editwin_for_count_lines(self, editwin, + linewidth, tabwidth): + """Prepare a mock EditorWindow object for Squeezer.count_lines.""" + CHAR_WIDTH = 10 + BORDER_WIDTH = 2 + PADDING_WIDTH = 1 + + # Prepare all the required functionality on the mock EditorWindow object + # so that the calculations in Squeezer.count_lines() can run. + editwin.get_tk_tabwidth.return_value = tabwidth + editwin.text.winfo_width.return_value = \ + linewidth * CHAR_WIDTH + 2 * (BORDER_WIDTH + PADDING_WIDTH) + text_opts = { + 'border': BORDER_WIDTH, + 'padx': PADDING_WIDTH, + 'font': None, + } + editwin.text.cget = lambda opt: text_opts[opt] + + # monkey-path tkinter.font.Font with a mock object, so that + # Font.measure('0') returns CHAR_WIDTH + mock_font = Mock() + def measure(char): + if char == '0': + return CHAR_WIDTH + raise ValueError("measure should only be called on '0'!") + mock_font.return_value.measure = measure + patcher = patch('idlelib.squeezer.Font', mock_font) + patcher.start() + self.addCleanup(patcher.stop) + + def _test_count_lines_helper(self, linewidth, tabwidth): + """Helper for test_count_lines.""" + editwin = self.make_mock_editor_window() + self._prepare_mock_editwin_for_count_lines(editwin, linewidth, tabwidth) + squeezer = self.make_squeezer_instance(editwin) + + mock_count_lines = Mock(return_value=SENTINEL_VALUE) + text = 'TEXT' + with patch('idlelib.squeezer.count_lines_with_wrapping', + mock_count_lines): + self.assertIs(squeezer.count_lines(text), SENTINEL_VALUE) + mock_count_lines.assert_called_with(text, linewidth, tabwidth) + + def test_init(self): + """Test the creation of Squeezer instances.""" + editwin = self.make_mock_editor_window() + squeezer = self.make_squeezer_instance(editwin) + self.assertIs(squeezer.editwin, editwin) + self.assertEqual(squeezer.expandingbuttons, []) + + def test_write_no_tags(self): + """Test Squeezer's overriding of the EditorWindow's write() method.""" + editwin = self.make_mock_editor_window() + for text in ['', 'TEXT', 'LONG TEXT' * 1000, 'MANY_LINES\n' * 100]: + editwin.write = orig_write = Mock(return_value=SENTINEL_VALUE) + squeezer = self.make_squeezer_instance(editwin) + + self.assertEqual(squeezer.editwin.write(text, ()), SENTINEL_VALUE) + self.assertEqual(orig_write.call_count, 1) + orig_write.assert_called_with(text, ()) + self.assertEqual(len(squeezer.expandingbuttons), 0) + + def test_write_not_stdout(self): + """Test Squeezer's overriding of the EditorWindow's write() method.""" + for text in ['', 'TEXT', 'LONG TEXT' * 1000, 'MANY_LINES\n' * 100]: + editwin = self.make_mock_editor_window() + editwin.write.return_value = SENTINEL_VALUE + orig_write = editwin.write + squeezer = self.make_squeezer_instance(editwin) + + self.assertEqual(squeezer.editwin.write(text, "stderr"), + SENTINEL_VALUE) + self.assertEqual(orig_write.call_count, 1) + orig_write.assert_called_with(text, "stderr") + self.assertEqual(len(squeezer.expandingbuttons), 0) + + def test_write_stdout(self): + """Test Squeezer's overriding of the EditorWindow's write() method.""" + editwin = self.make_mock_editor_window() + self._prepare_mock_editwin_for_count_lines(editwin, + linewidth=80, tabwidth=8) + + for text in ['', 'TEXT']: + editwin.write = orig_write = Mock(return_value=SENTINEL_VALUE) + squeezer = self.make_squeezer_instance(editwin) + squeezer.auto_squeeze_min_lines = 50 + + self.assertEqual(squeezer.editwin.write(text, "stdout"), + SENTINEL_VALUE) + self.assertEqual(orig_write.call_count, 1) + orig_write.assert_called_with(text, "stdout") + self.assertEqual(len(squeezer.expandingbuttons), 0) + + for text in ['LONG TEXT' * 1000, 'MANY_LINES\n' * 100]: + editwin.write = orig_write = Mock(return_value=SENTINEL_VALUE) + squeezer = self.make_squeezer_instance(editwin) + squeezer.auto_squeeze_min_lines = 50 + + self.assertEqual(squeezer.editwin.write(text, "stdout"), None) + self.assertEqual(orig_write.call_count, 0) + self.assertEqual(len(squeezer.expandingbuttons), 1) + + def test_auto_squeeze(self): + """Test that the auto-squeezing creates an ExpandingButton properly.""" + root = get_test_tk_root(self) + text_widget = Text(root) + text_widget.mark_set("iomark", "1.0") + + editwin = self.make_mock_editor_window() + editwin.text = text_widget + squeezer = self.make_squeezer_instance(editwin) + squeezer.auto_squeeze_min_lines = 5 + squeezer.count_lines = Mock(return_value=6) + + editwin.write('TEXT\n'*6, "stdout") + self.assertEqual(text_widget.get('1.0', 'end'), '\n') + self.assertEqual(len(squeezer.expandingbuttons), 1) + + def test_squeeze_current_text_event(self): + """Test the squeeze_current_text event.""" + root = get_test_tk_root(self) + + # squeezing text should work for both stdout and stderr + for tag_name in ["stdout", "stderr"]: + text_widget = Text(root) + text_widget.mark_set("iomark", "1.0") + + editwin = self.make_mock_editor_window() + editwin.text = editwin.per.bottom = text_widget + squeezer = self.make_squeezer_instance(editwin) + squeezer.count_lines = Mock(return_value=6) + + # prepare some text in the Text widget + text_widget.insert("1.0", "SOME\nTEXT\n", tag_name) + text_widget.mark_set("insert", "1.0") + self.assertEqual(text_widget.get('1.0', 'end'), 'SOME\nTEXT\n\n') + + self.assertEqual(len(squeezer.expandingbuttons), 0) + + # test squeezing the current text + retval = squeezer.squeeze_current_text_event(event=Mock()) + self.assertEqual(retval, "break") + self.assertEqual(text_widget.get('1.0', 'end'), '\n\n') + self.assertEqual(len(squeezer.expandingbuttons), 1) + self.assertEqual(squeezer.expandingbuttons[0].s, 'SOME\nTEXT') + + # test that expanding the squeezed text works and afterwards the + # Text widget contains the original text + squeezer.expandingbuttons[0].expand(event=Mock()) + self.assertEqual(text_widget.get('1.0', 'end'), 'SOME\nTEXT\n\n') + self.assertEqual(len(squeezer.expandingbuttons), 0) + + def test_squeeze_current_text_event_no_allowed_tags(self): + """Test that the event doesn't squeeze text without a relevant tag.""" + root = get_test_tk_root(self) + + text_widget = Text(root) + text_widget.mark_set("iomark", "1.0") + + editwin = self.make_mock_editor_window() + editwin.text = editwin.per.bottom = text_widget + squeezer = self.make_squeezer_instance(editwin) + squeezer.count_lines = Mock(return_value=6) + + # prepare some text in the Text widget + text_widget.insert("1.0", "SOME\nTEXT\n", "TAG") + text_widget.mark_set("insert", "1.0") + self.assertEqual(text_widget.get('1.0', 'end'), 'SOME\nTEXT\n\n') + + self.assertEqual(len(squeezer.expandingbuttons), 0) + + # test squeezing the current text + retval = squeezer.squeeze_current_text_event(event=Mock()) + self.assertEqual(retval, "break") + self.assertEqual(text_widget.get('1.0', 'end'), 'SOME\nTEXT\n\n') + self.assertEqual(len(squeezer.expandingbuttons), 0) + + def test_squeeze_text_before_existing_squeezed_text(self): + """Test squeezing text before existing squeezed text.""" + root = get_test_tk_root(self) + + text_widget = Text(root) + text_widget.mark_set("iomark", "1.0") + + editwin = self.make_mock_editor_window() + editwin.text = editwin.per.bottom = text_widget + squeezer = self.make_squeezer_instance(editwin) + squeezer.count_lines = Mock(return_value=6) + + # prepare some text in the Text widget and squeeze it + text_widget.insert("1.0", "SOME\nTEXT\n", "stdout") + text_widget.mark_set("insert", "1.0") + squeezer.squeeze_current_text_event(event=Mock()) + self.assertEqual(len(squeezer.expandingbuttons), 1) + + # test squeezing the current text + text_widget.insert("1.0", "MORE\nSTUFF\n", "stdout") + text_widget.mark_set("insert", "1.0") + retval = squeezer.squeeze_current_text_event(event=Mock()) + self.assertEqual(retval, "break") + self.assertEqual(text_widget.get('1.0', 'end'), '\n\n\n') + self.assertEqual(len(squeezer.expandingbuttons), 2) + self.assertTrue(text_widget.compare( + squeezer.expandingbuttons[0], + '<', + squeezer.expandingbuttons[1], + )) + + GetOptionSignature = namedtuple('GetOptionSignature', + 'configType section option default type warn_on_default raw') + @classmethod + def _make_sig(cls, configType, section, option, default=sentinel.NOT_GIVEN, + type=sentinel.NOT_GIVEN, + warn_on_default=sentinel.NOT_GIVEN, + raw=sentinel.NOT_GIVEN): + return cls.GetOptionSignature(configType, section, option, default, + type, warn_on_default, raw) + + @classmethod + def get_GetOption_signature(cls, mock_call_obj): + args, kwargs = mock_call_obj[-2:] + return cls._make_sig(*args, **kwargs) + + def test_reload(self): + """Test the reload() class-method.""" + self.assertIsInstance(Squeezer.auto_squeeze_min_lines, int) + idleConf.SetOption('main', 'PyShell', 'auto-squeeze-min-lines', '42') + Squeezer.reload() + self.assertEqual(Squeezer.auto_squeeze_min_lines, 42) + + +class ExpandingButtonTest(unittest.TestCase): + """Tests for the ExpandingButton class.""" + # In these tests the squeezer instance is a mock, but actual tkinter + # Text and Button instances are created. + def make_mock_squeezer(self): + """Helper for tests: Create a mock Squeezer object.""" + root = get_test_tk_root(self) + squeezer = Mock() + squeezer.editwin.text = Text(root) + + # Set default values for the configuration settings + squeezer.auto_squeeze_min_lines = 50 + return squeezer + + @patch('idlelib.squeezer.Hovertip', autospec=Hovertip) + def test_init(self, MockHovertip): + """Test the simplest creation of an ExpandingButton.""" + squeezer = self.make_mock_squeezer() + text_widget = squeezer.editwin.text + + expandingbutton = ExpandingButton('TEXT', 'TAGS', 50, squeezer) + self.assertEqual(expandingbutton.s, 'TEXT') + + # check that the underlying tkinter.Button is properly configured + self.assertEqual(expandingbutton.master, text_widget) + self.assertTrue('50 lines' in expandingbutton.cget('text')) + + # check that the text widget still contains no text + self.assertEqual(text_widget.get('1.0', 'end'), '\n') + + # check that the mouse events are bound + self.assertIn('', expandingbutton.bind()) + right_button_code = '' % ('2' if macosx.isAquaTk() else '3') + self.assertIn(right_button_code, expandingbutton.bind()) + + # check that ToolTip was called once, with appropriate values + self.assertEqual(MockHovertip.call_count, 1) + MockHovertip.assert_called_with(expandingbutton, ANY, hover_delay=ANY) + + # check that 'right-click' appears in the tooltip text + tooltip_text = MockHovertip.call_args[0][1] + self.assertIn('right-click', tooltip_text.lower()) + + def test_expand(self): + """Test the expand event.""" + squeezer = self.make_mock_squeezer() + expandingbutton = ExpandingButton('TEXT', 'TAGS', 50, squeezer) + + # insert the button into the text widget + # (this is normally done by the Squeezer class) + text_widget = expandingbutton.text + text_widget.window_create("1.0", window=expandingbutton) + + # set base_text to the text widget, so that changes are actually made + # to it (by ExpandingButton) and we can inspect these changes afterwards + expandingbutton.base_text = expandingbutton.text + + # trigger the expand event + retval = expandingbutton.expand(event=Mock()) + self.assertEqual(retval, None) + + # check that the text was inserted into the text widget + self.assertEqual(text_widget.get('1.0', 'end'), 'TEXT\n') + + # check that the 'TAGS' tag was set on the inserted text + text_end_index = text_widget.index('end-1c') + self.assertEqual(text_widget.get('1.0', text_end_index), 'TEXT') + self.assertEqual(text_widget.tag_nextrange('TAGS', '1.0'), + ('1.0', text_end_index)) + + # check that the button removed itself from squeezer.expandingbuttons + self.assertEqual(squeezer.expandingbuttons.remove.call_count, 1) + squeezer.expandingbuttons.remove.assert_called_with(expandingbutton) + + def test_expand_dangerous_oupput(self): + """Test that expanding very long output asks user for confirmation.""" + squeezer = self.make_mock_squeezer() + text = 'a' * 10**5 + expandingbutton = ExpandingButton(text, 'TAGS', 50, squeezer) + expandingbutton.set_is_dangerous() + self.assertTrue(expandingbutton.is_dangerous) + + # insert the button into the text widget + # (this is normally done by the Squeezer class) + text_widget = expandingbutton.text + text_widget.window_create("1.0", window=expandingbutton) + + # set base_text to the text widget, so that changes are actually made + # to it (by ExpandingButton) and we can inspect these changes afterwards + expandingbutton.base_text = expandingbutton.text + + # patch the message box module to always return False + with patch('idlelib.squeezer.tkMessageBox') as mock_msgbox: + mock_msgbox.askokcancel.return_value = False + mock_msgbox.askyesno.return_value = False + + # trigger the expand event + retval = expandingbutton.expand(event=Mock()) + + # check that the event chain was broken and no text was inserted + self.assertEqual(retval, 'break') + self.assertEqual(expandingbutton.text.get('1.0', 'end-1c'), '') + + # patch the message box module to always return True + with patch('idlelib.squeezer.tkMessageBox') as mock_msgbox: + mock_msgbox.askokcancel.return_value = True + mock_msgbox.askyesno.return_value = True + + # trigger the expand event + retval = expandingbutton.expand(event=Mock()) + + # check that the event chain wasn't broken and the text was inserted + self.assertEqual(retval, None) + self.assertEqual(expandingbutton.text.get('1.0', 'end-1c'), text) + + def test_copy(self): + """Test the copy event.""" + # testing with the actual clipboard proved problematic, so this test + # replaces the clipboard manipulation functions with mocks and checks + # that they are called appropriately + squeezer = self.make_mock_squeezer() + expandingbutton = ExpandingButton('TEXT', 'TAGS', 50, squeezer) + expandingbutton.clipboard_clear = Mock() + expandingbutton.clipboard_append = Mock() + + # trigger the copy event + retval = expandingbutton.copy(event=Mock()) + self.assertEqual(retval, None) + + # check that the expanding button called clipboard_clear() and + # clipboard_append('TEXT') once each + self.assertEqual(expandingbutton.clipboard_clear.call_count, 1) + self.assertEqual(expandingbutton.clipboard_append.call_count, 1) + expandingbutton.clipboard_append.assert_called_with('TEXT') + + def test_view(self): + """Test the view event.""" + squeezer = self.make_mock_squeezer() + expandingbutton = ExpandingButton('TEXT', 'TAGS', 50, squeezer) + expandingbutton.selection_own = Mock() + + with patch('idlelib.squeezer.view_text', autospec=view_text)\ + as mock_view_text: + # trigger the view event + expandingbutton.view(event=Mock()) + + # check that the expanding button called view_text + self.assertEqual(mock_view_text.call_count, 1) + + # check that the proper text was passed + self.assertEqual(mock_view_text.call_args[0][2], 'TEXT') + + def test_rmenu(self): + """Test the context menu.""" + squeezer = self.make_mock_squeezer() + expandingbutton = ExpandingButton('TEXT', 'TAGS', 50, squeezer) + with patch('tkinter.Menu') as mock_Menu: + mock_menu = Mock() + mock_Menu.return_value = mock_menu + mock_event = Mock() + mock_event.x = 10 + mock_event.y = 10 + expandingbutton.context_menu_event(event=mock_event) + self.assertEqual(mock_menu.add_command.call_count, + len(expandingbutton.rmenu_specs)) + for label, *data in expandingbutton.rmenu_specs: + mock_menu.add_command.assert_any_call(label=label, command=ANY) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_stackviewer.py python3.7-3.7.1/Lib/idlelib/idle_test/test_stackviewer.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_stackviewer.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_stackviewer.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,47 @@ +"Test stackviewer, coverage 63%." + +from idlelib import stackviewer +import unittest +from test.support import requires +from tkinter import Tk + +from idlelib.tree import TreeNode, ScrolledCanvas +import sys + + +class StackBrowserTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + svs = stackviewer.sys + try: + abc + except NameError: + svs.last_type, svs.last_value, svs.last_traceback = ( + sys.exc_info()) + + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + svs = stackviewer.sys + del svs.last_traceback, svs.last_type, svs.last_value + + cls.root.update_idletasks() +## for id in cls.root.tk.call('after', 'info'): +## cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_init(self): + sb = stackviewer.StackBrowser(self.root) + isi = self.assertIsInstance + isi(stackviewer.sc, ScrolledCanvas) + isi(stackviewer.item, stackviewer.StackTreeItem) + isi(stackviewer.node, TreeNode) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_statusbar.py python3.7-3.7.1/Lib/idlelib/idle_test/test_statusbar.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_statusbar.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_statusbar.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,41 @@ +"Test statusbar, coverage 100%." + +from idlelib import statusbar +import unittest +from test.support import requires +from tkinter import Tk + + +class Test(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_init(self): + bar = statusbar.MultiStatusBar(self.root) + self.assertEqual(bar.labels, {}) + + def test_set_label(self): + bar = statusbar.MultiStatusBar(self.root) + bar.set_label('left', text='sometext', width=10) + self.assertIn('left', bar.labels) + left = bar.labels['left'] + self.assertEqual(left['text'], 'sometext') + self.assertEqual(left['width'], 10) + bar.set_label('left', text='revised text') + self.assertEqual(left['text'], 'revised text') + bar.set_label('right', text='correct text') + self.assertEqual(bar.labels['right']['text'], 'correct text') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_textview.py python3.7-3.7.1/Lib/idlelib/idle_test/test_textview.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_textview.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_textview.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,17 +1,15 @@ -'''Test idlelib.textview. +"""Test textview, coverage 100%. Since all methods and functions create (or destroy) a ViewWindow, which is a widget containing a widget, etcetera, all tests must be gui tests. Using mock Text would not change this. Other mocks are used to retrieve information about calls. - -Coverage: 100%. -''' +""" from idlelib import textview as tv +import unittest from test.support import requires requires('gui') -import unittest import os from tkinter import Tk from tkinter.ttk import Button @@ -75,7 +73,6 @@ @classmethod def setUpClass(cls): - "By itself, this tests that file parsed without exception." cls.root = root = Tk() root.withdraw() cls.frame = tv.TextFrame(root, 'test text') @@ -109,7 +106,7 @@ view = tv.view_text(root, 'Title', 'test text', modal=False) self.assertIsInstance(view, tv.ViewWindow) self.assertIsInstance(view.viewframe, tv.ViewFrame) - view.ok() + view.viewframe.ok() def test_view_file(self): view = tv.view_file(root, 'Title', __file__, 'ascii', modal=False) @@ -128,11 +125,15 @@ def test_bad_encoding(self): p = os.path fn = p.abspath(p.join(p.dirname(__file__), '..', 'CREDITS.txt')) - tv.showerror.title = None view = tv.view_file(root, 'Title', fn, 'ascii', modal=False) self.assertIsNone(view) self.assertEqual(tv.showerror.title, 'Unicode Decode Error') + def test_nowrap(self): + view = tv.view_text(root, 'Title', 'test', modal=False, wrap='none') + text_widget = view.viewframe.textframe.text + self.assertEqual(text_widget.cget('wrap'), 'none') + # Call ViewWindow with _utest=True. class ButtonClickTest(unittest.TestCase): diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_tooltip.py python3.7-3.7.1/Lib/idlelib/idle_test/test_tooltip.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_tooltip.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_tooltip.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,146 @@ +from idlelib.tooltip import TooltipBase, Hovertip +from test.support import requires +requires('gui') + +from functools import wraps +import time +from tkinter import Button, Tk, Toplevel +import unittest + + +def setUpModule(): + global root + root = Tk() + +def root_update(): + global root + root.update() + +def tearDownModule(): + global root + root.update_idletasks() + root.destroy() + del root + +def add_call_counting(func): + @wraps(func) + def wrapped_func(*args, **kwargs): + wrapped_func.call_args_list.append((args, kwargs)) + return func(*args, **kwargs) + wrapped_func.call_args_list = [] + return wrapped_func + + +def _make_top_and_button(testobj): + global root + top = Toplevel(root) + testobj.addCleanup(top.destroy) + top.title("Test tooltip") + button = Button(top, text='ToolTip test button') + button.pack() + testobj.addCleanup(button.destroy) + top.lift() + return top, button + + +class ToolTipBaseTest(unittest.TestCase): + def setUp(self): + self.top, self.button = _make_top_and_button(self) + + def test_base_class_is_unusable(self): + global root + top = Toplevel(root) + self.addCleanup(top.destroy) + + button = Button(top, text='ToolTip test button') + button.pack() + self.addCleanup(button.destroy) + + with self.assertRaises(NotImplementedError): + tooltip = TooltipBase(button) + tooltip.showtip() + + +class HovertipTest(unittest.TestCase): + def setUp(self): + self.top, self.button = _make_top_and_button(self) + + def test_showtip(self): + tooltip = Hovertip(self.button, 'ToolTip text') + self.addCleanup(tooltip.hidetip) + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + tooltip.showtip() + self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + + def test_showtip_twice(self): + tooltip = Hovertip(self.button, 'ToolTip text') + self.addCleanup(tooltip.hidetip) + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + tooltip.showtip() + self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + orig_tipwindow = tooltip.tipwindow + tooltip.showtip() + self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertIs(tooltip.tipwindow, orig_tipwindow) + + def test_hidetip(self): + tooltip = Hovertip(self.button, 'ToolTip text') + self.addCleanup(tooltip.hidetip) + tooltip.showtip() + tooltip.hidetip() + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + + def test_showtip_on_mouse_enter_no_delay(self): + tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None) + self.addCleanup(tooltip.hidetip) + tooltip.showtip = add_call_counting(tooltip.showtip) + root_update() + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.button.event_generate('', x=0, y=0) + root_update() + self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertGreater(len(tooltip.showtip.call_args_list), 0) + + def test_showtip_on_mouse_enter_hover_delay(self): + tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50) + self.addCleanup(tooltip.hidetip) + tooltip.showtip = add_call_counting(tooltip.showtip) + root_update() + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.button.event_generate('', x=0, y=0) + root_update() + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + time.sleep(0.1) + root_update() + self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertGreater(len(tooltip.showtip.call_args_list), 0) + + def test_hidetip_on_mouse_leave(self): + tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None) + self.addCleanup(tooltip.hidetip) + tooltip.showtip = add_call_counting(tooltip.showtip) + root_update() + self.button.event_generate('', x=0, y=0) + root_update() + self.button.event_generate('', x=0, y=0) + root_update() + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertGreater(len(tooltip.showtip.call_args_list), 0) + + def test_dont_show_on_mouse_leave_before_delay(self): + tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50) + self.addCleanup(tooltip.hidetip) + tooltip.showtip = add_call_counting(tooltip.showtip) + root_update() + self.button.event_generate('', x=0, y=0) + root_update() + self.button.event_generate('', x=0, y=0) + root_update() + time.sleep(0.1) + root_update() + self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertEqual(tooltip.showtip.call_args_list, []) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_tree.py python3.7-3.7.1/Lib/idlelib/idle_test/test_tree.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_tree.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_tree.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,11 +1,9 @@ -''' Test idlelib.tree. +"Test tree. coverage 56%." -Coverage: 56% -''' from idlelib import tree +import unittest from test.support import requires requires('gui') -import unittest from tkinter import Tk diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_undo.py python3.7-3.7.1/Lib/idlelib/idle_test/test_undo.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_undo.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_undo.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,14 +1,13 @@ -"""Unittest for UndoDelegator in idlelib.undo.py. +"Test undo, coverage 77%." +# Only test UndoDelegator so far. -Coverage about 80% (retest). -""" +from idlelib.undo import UndoDelegator +import unittest from test.support import requires requires('gui') -import unittest from unittest.mock import Mock from tkinter import Text, Tk -from idlelib.undo import UndoDelegator from idlelib.percolator import Percolator @@ -131,5 +130,6 @@ text.insert('insert', 'foo') self.assertLessEqual(len(self.delegator.undolist), max_undo) + if __name__ == '__main__': unittest.main(verbosity=2, exit=False) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_warning.py python3.7-3.7.1/Lib/idlelib/idle_test/test_warning.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_warning.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_warning.py 2018-10-20 06:04:19.000000000 +0000 @@ -5,20 +5,18 @@ Revise if output destination changes (http://bugs.python.org/issue18318). Make sure warnings module is left unaltered (http://bugs.python.org/issue18081). ''' - +from idlelib import run +from idlelib import pyshell as shell import unittest from test.support import captured_stderr - import warnings + # Try to capture default showwarning before Idle modules are imported. showwarning = warnings.showwarning # But if we run this file within idle, we are in the middle of the run.main loop # and default showwarnings has already been replaced. running_in_idle = 'idle' in showwarning.__name__ -from idlelib import run -from idlelib import pyshell as shell - # The following was generated from pyshell.idle_formatwarning # and checked as matching expectation. idlemsg = ''' @@ -29,6 +27,7 @@ ''' shellmsg = idlemsg + ">>> " + class RunWarnTest(unittest.TestCase): @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") @@ -46,6 +45,7 @@ # The following uses .splitlines to erase line-ending differences self.assertEqual(idlemsg.splitlines(), f.getvalue().splitlines()) + class ShellWarnTest(unittest.TestCase): @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") @@ -70,4 +70,4 @@ if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_window.py python3.7-3.7.1/Lib/idlelib/idle_test/test_window.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_window.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_window.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,45 @@ +"Test window, coverage 47%." + +from idlelib import window +import unittest +from test.support import requires +from tkinter import Tk + + +class WindowListTest(unittest.TestCase): + + def test_init(self): + wl = window.WindowList() + self.assertEqual(wl.dict, {}) + self.assertEqual(wl.callbacks, []) + + # Further tests need mock Window. + + +class ListedToplevelTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + window.registry = set() + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + window.registry = window.WindowList() + cls.root.update_idletasks() +## for id in cls.root.tk.call('after', 'info'): +## cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_init(self): + + win = window.ListedToplevel(self.root) + self.assertIn(win, window.registry) + self.assertEqual(win.focused_widget, win) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/idle_test/test_zoomheight.py python3.7-3.7.1/Lib/idlelib/idle_test/test_zoomheight.py --- python3.7-3.7.0/Lib/idlelib/idle_test/test_zoomheight.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/idle_test/test_zoomheight.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,39 @@ +"Test zoomheight, coverage 66%." +# Some code is system dependent. + +from idlelib import zoomheight +import unittest +from test.support import requires +from tkinter import Tk, Text +from idlelib.editor import EditorWindow + + +class Test(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.editwin = EditorWindow(root=cls.root) + + @classmethod + def tearDownClass(cls): + cls.editwin._close() + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def test_init(self): + zoom = zoomheight.ZoomHeight(self.editwin) + self.assertIs(zoom.editwin, self.editwin) + + def test_zoom_height_event(self): + zoom = zoomheight.ZoomHeight(self.editwin) + zoom.zoom_height_event() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/iomenu.py python3.7-3.7.1/Lib/idlelib/iomenu.py --- python3.7-3.7.0/Lib/idlelib/iomenu.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/iomenu.py 2018-10-20 06:04:19.000000000 +0000 @@ -40,7 +40,7 @@ # resulting codeset may be unknown to Python. We ignore all # these problems, falling back to ASCII locale_encoding = locale.nl_langinfo(locale.CODESET) - if locale_encoding is None or locale_encoding is '': + if locale_encoding is None or locale_encoding == '': # situation occurs on Mac OS X locale_encoding = 'ascii' codecs.lookup(locale_encoding) @@ -50,7 +50,7 @@ # bugs that can cause ValueError. try: locale_encoding = locale.getdefaultlocale()[1] - if locale_encoding is None or locale_encoding is '': + if locale_encoding is None or locale_encoding == '': # situation occurs on Mac OS X locale_encoding = 'ascii' codecs.lookup(locale_encoding) @@ -567,8 +567,8 @@ IOBinding(editwin) if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_iomenu', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_iomenu', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_io_binding) diff -Nru python3.7-3.7.0/Lib/idlelib/macosx.py python3.7-3.7.1/Lib/idlelib/macosx.py --- python3.7-3.7.0/Lib/idlelib/macosx.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/macosx.py 2018-10-20 06:04:19.000000000 +0000 @@ -128,7 +128,7 @@ # menu. from tkinter import Menu from idlelib import mainmenu - from idlelib import windows + from idlelib import window closeItem = mainmenu.menudefs[0][1][-2] @@ -148,7 +148,7 @@ root.configure(menu=menubar) menudict = {} - menudict['windows'] = menu = Menu(menubar, name='windows', tearoff=0) + menudict['window'] = menu = Menu(menubar, name='window', tearoff=0) menubar.add_cascade(label='Window', menu=menu, underline=0) def postwindowsmenu(menu=menu): @@ -158,8 +158,8 @@ if end > 0: menu.delete(0, end) - windows.add_windows_to_menu(menu) - windows.register_callback(postwindowsmenu) + window.add_windows_to_menu(menu) + window.register_callback(postwindowsmenu) def about_dialog(event=None): "Handle Help 'About IDLE' event." diff -Nru python3.7-3.7.0/Lib/idlelib/mainmenu.py python3.7-3.7.1/Lib/idlelib/mainmenu.py --- python3.7-3.7.0/Lib/idlelib/mainmenu.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/mainmenu.py 2018-10-20 06:04:19.000000000 +0000 @@ -36,7 +36,8 @@ None, ('_Close', '<>'), ('E_xit', '<>'), - ]), + ]), + ('edit', [ ('_Undo', '<>'), ('_Redo', '<>'), @@ -56,9 +57,9 @@ ('E_xpand Word', '<>'), ('Show C_all Tip', '<>'), ('Show Surrounding P_arens', '<>'), + ]), - ]), -('format', [ + ('format', [ ('_Indent Region', '<>'), ('_Dedent Region', '<>'), ('Comment _Out Region', '<>'), @@ -70,30 +71,36 @@ ('F_ormat Paragraph', '<>'), ('S_trip Trailing Whitespace', '<>'), ]), + ('run', [ ('Python Shell', '<>'), ('C_heck Module', '<>'), ('R_un Module', '<>'), ]), + ('shell', [ ('_View Last Restart', '<>'), ('_Restart Shell', '<>'), None, ('_Interrupt Execution', '<>'), ]), + ('debug', [ ('_Go to File/Line', '<>'), ('!_Debugger', '<>'), ('_Stack Viewer', '<>'), ('!_Auto-open Stack Viewer', '<>'), ]), + ('options', [ ('Configure _IDLE', '<>'), ('_Code Context', '<>'), ]), - ('windows', [ + + ('window', [ ('Zoom Height', '<>'), ]), + ('help', [ ('_About IDLE', '<>'), None, @@ -106,3 +113,7 @@ menudefs[-1][1].append(('Turtle Demo', '<>')) default_keydefs = idleConf.GetCurrentKeySet() + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_mainmenu', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/multicall.py python3.7-3.7.1/Lib/idlelib/multicall.py --- python3.7-3.7.0/Lib/idlelib/multicall.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/multicall.py 2018-10-20 06:04:19.000000000 +0000 @@ -441,5 +441,8 @@ bindseq("") if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_mainmenu', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_multi_call) diff -Nru python3.7-3.7.0/Lib/idlelib/NEWS.txt python3.7-3.7.1/Lib/idlelib/NEWS.txt --- python3.7-3.7.0/Lib/idlelib/NEWS.txt 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/NEWS.txt 2018-10-20 06:04:19.000000000 +0000 @@ -1,8 +1,65 @@ -What's New in IDLE 3.7.0 (since 3.6.0) -Released on 2018-06-18? +What's New in IDLE 3.7.1 +Released on 2018-07-31? ====================================== +bpo-1529353: Output over N lines (50 by default) is squeezed down to a button. +N can be changed in the PyShell section of the General page of the +Settings dialog. Fewer, but possibly extra long, lines can be squeezed by +right clicking on the output. Squeezed output can be expanded in place +by double-clicking the button or into the clipboard or a separate window +by right-clicking the button. + +bpo-34548: Use configured color theme for read-only text views. + +bpo-33839: Refactor ToolTip and CallTip classes; add documentation +and tests. + +bpo-34047: Fix mouse wheel scrolling direction on macOS. + +bpo-34275: Make calltips always visible on Mac. +Patch by Kevin Walzer. + +bpo-34120: Fix freezing after closing some dialogs on Mac. +This is one of multiple regressions from using newer tcl/tk. + +bpo-33975: Avoid small type when running htests. +Since part of the purpose of human-viewed tests is to determine that +widgets look right, it is important that they look the same for +testing as when running IDLE. + +bpo-33905: Add test for idlelib.stackview.StackBrowser. + +bpo-33924: Change mainmenu.menudefs key 'windows' to 'window'. +Every other menudef key is the lowercase version of the +corresponding main menu entry (in this case, 'Window'). + +bpo-33906: Rename idlelib.windows as window +Match Window on the main menu and remove last plural module name. +Change imports, test, and attribute references to match new name. + +bpo-33917: Fix and document idlelib/idle_test/template.py. +The revised file compiles, runs, and tests OK. idle_test/README.txt +explains how to use it to create new IDLE test files. + +bpo-33904: In rstrip module, rename class RstripExtension as Rstrip. + +bpo-33907: For consistency and clarity, rename calltip objects. +Module calltips and its class CallTips are now calltip and Calltip. +In module calltip_w, class CallTip is now CalltipWindow. + +bpo-33855: Minimally test all IDLE modules. +Standardize the test file format. Add missing test files that import +the tested module and perform at least one test. Check and record the +coverage of each test. + +bpo-33856: Add 'help' to Shell's initial welcome message. + + +What's New in IDLE 3.7.0 (since 3.6.0) +Released on 2018-06-27 +====================================== + bpo-33656: On Windows, add API call saying that tk scales for DPI. On Windows 8.1+ or 10, with DPI compatibility properties of the Python binary unchanged, and a monitor resolution greater than 96 DPI, this @@ -165,7 +222,7 @@ bpo-31421: Document how IDLE runs tkinter programs. IDLE calls tcl/tk update in the background in order to make live -interaction and experimentatin with tkinter applications much easier. +interaction and experimentation with tkinter applications much easier. bpo-31414: Fix tk entry box tests by deleting first. Adding to an int entry is not the same as deleting and inserting @@ -403,7 +460,7 @@ -w option but without -jn. Fix warning from test_config. - Issue #27621: Put query response validation error messages in the query - box itself instead of in a separate massagebox. Redo tests to match. + box itself instead of in a separate messagebox. Redo tests to match. Add Mac OSX refinements. Original patch by Mark Roseman. - Issue #27620: Escape key now closes Query box as cancelled. @@ -469,7 +526,7 @@ - Issue #27239: idlelib.macosx.isXyzTk functions initialize as needed. -- Issue #27262: move Aqua unbinding code, which enable context menus, to maxosx. +- Issue #27262: move Aqua unbinding code, which enable context menus, to macosx. - Issue #24759: Make clear in idlelib.idle_test.__init__ that the directory is a private implementation of test.test_idle and tool for maintainers. diff -Nru python3.7-3.7.0/Lib/idlelib/outwin.py python3.7-3.7.1/Lib/idlelib/outwin.py --- python3.7-3.7.0/Lib/idlelib/outwin.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/outwin.py 2018-10-20 06:04:19.000000000 +0000 @@ -184,5 +184,5 @@ self.write = self.owin.write if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_outwin', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_outwin', verbosity=2, exit=False) diff -Nru python3.7-3.7.0/Lib/idlelib/paragraph.py python3.7-3.7.1/Lib/idlelib/paragraph.py --- python3.7-3.7.0/Lib/idlelib/paragraph.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/paragraph.py 2018-10-20 06:04:19.000000000 +0000 @@ -190,6 +190,5 @@ if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_paragraph', - verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_paragraph', verbosity=2, exit=False) diff -Nru python3.7-3.7.0/Lib/idlelib/parenmatch.py python3.7-3.7.1/Lib/idlelib/parenmatch.py --- python3.7-3.7.0/Lib/idlelib/parenmatch.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/parenmatch.py 2018-10-20 06:04:19.000000000 +0000 @@ -179,5 +179,5 @@ if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_parenmatch', verbosity=2) + from unittest import main + main('idlelib.idle_test.test_parenmatch', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/percolator.py python3.7-3.7.1/Lib/idlelib/percolator.py --- python3.7-3.7.0/Lib/idlelib/percolator.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/percolator.py 2018-10-20 06:04:19.000000000 +0000 @@ -96,9 +96,8 @@ cb2.pack() if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_percolator', verbosity=2, - exit=False) + from unittest import main + main('idlelib.idle_test.test_percolator', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_percolator) diff -Nru python3.7-3.7.0/Lib/idlelib/pyparse.py python3.7-3.7.1/Lib/idlelib/pyparse.py --- python3.7-3.7.0/Lib/idlelib/pyparse.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/pyparse.py 2018-10-20 06:04:19.000000000 +0000 @@ -594,6 +594,6 @@ return self.stmt_bracketing -if __name__ == '__main__': #pragma: nocover - import unittest - unittest.main('idlelib.idle_test.test_pyparse', verbosity=2) +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_pyparse', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/pyshell.py python3.7-3.7.1/Lib/idlelib/pyshell.py --- python3.7-3.7.0/Lib/idlelib/pyshell.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/pyshell.py 2018-10-20 06:04:19.000000000 +0000 @@ -852,10 +852,14 @@ ("edit", "_Edit"), ("debug", "_Debug"), ("options", "_Options"), - ("windows", "_Window"), + ("window", "_Window"), ("help", "_Help"), ] + # Extend right-click context menu + rmenu_specs = OutputWindow.rmenu_specs + [ + ("Squeeze", "<>"), + ] # New classes from idlelib.history import History @@ -1033,7 +1037,7 @@ return self.shell_title COPYRIGHT = \ - 'Type "copyright", "credits" or "license()" for more information.' + 'Type "help", "copyright", "credits" or "license()" for more information.' def begin(self): self.text.mark_set("iomark", "insert") diff -Nru python3.7-3.7.0/Lib/idlelib/query.py python3.7-3.7.1/Lib/idlelib/query.py --- python3.7-3.7.0/Lib/idlelib/query.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/query.py 2018-10-20 06:04:19.000000000 +0000 @@ -143,6 +143,10 @@ self.result = None self.destroy() + def destroy(self): + self.grab_release() + super().destroy() + class SectionName(Query): "Get a name for a config file section name." @@ -301,8 +305,8 @@ if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_query', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_query', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(Query, HelpSource) diff -Nru python3.7-3.7.0/Lib/idlelib/redirector.py python3.7-3.7.1/Lib/idlelib/redirector.py --- python3.7-3.7.0/Lib/idlelib/redirector.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/redirector.py 2018-10-20 06:04:19.000000000 +0000 @@ -167,9 +167,8 @@ original_insert = redir.register("insert", my_insert) if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_redirector', - verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_redirector', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_widget_redirector) diff -Nru python3.7-3.7.0/Lib/idlelib/replace.py python3.7-3.7.1/Lib/idlelib/replace.py --- python3.7-3.7.0/Lib/idlelib/replace.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/replace.py 2018-10-20 06:04:19.000000000 +0000 @@ -235,9 +235,8 @@ button.pack() if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_replace', - verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_replace', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_replace_dialog) diff -Nru python3.7-3.7.0/Lib/idlelib/rpc.py python3.7-3.7.1/Lib/idlelib/rpc.py --- python3.7-3.7.0/Lib/idlelib/rpc.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/rpc.py 2018-10-20 06:04:19.000000000 +0000 @@ -43,16 +43,20 @@ import types def unpickle_code(ms): + "Return code object from marshal string ms." co = marshal.loads(ms) assert isinstance(co, types.CodeType) return co def pickle_code(co): + "Return unpickle function and tuple with marshalled co code object." assert isinstance(co, types.CodeType) ms = marshal.dumps(co) return unpickle_code, (ms,) def dumps(obj, protocol=None): + "Return pickled (or marshalled) string for obj." + # IDLE passes 'None' to select pickle.DEFAULT_PROTOCOL. f = io.BytesIO() p = CodePickler(f, protocol) p.dump(obj) @@ -625,3 +629,8 @@ sys.stdout.write(text) sys.stdout.write("\n") builtins._ = value + + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_rpc', verbosity=2,) diff -Nru python3.7-3.7.0/Lib/idlelib/rstrip.py python3.7-3.7.1/Lib/idlelib/rstrip.py --- python3.7-3.7.0/Lib/idlelib/rstrip.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/rstrip.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,6 +1,6 @@ 'Provides "Strip trailing whitespace" under the "Format" menu.' -class RstripExtension: +class Rstrip: def __init__(self, editwin): self.editwin = editwin @@ -25,5 +25,5 @@ undo.undo_block_stop() if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_rstrip', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_rstrip', verbosity=2,) diff -Nru python3.7-3.7.0/Lib/idlelib/run.py python3.7-3.7.1/Lib/idlelib/run.py --- python3.7-3.7.0/Lib/idlelib/run.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/run.py 2018-10-20 06:04:19.000000000 +0000 @@ -11,7 +11,7 @@ import tkinter # Tcl, deletions, messagebox if startup fails from idlelib import autocomplete # AutoComplete, fetch_encodings -from idlelib import calltips # CallTips +from idlelib import calltip # Calltip from idlelib import debugger_r # start_debugger from idlelib import debugobj_r # remote_object_tree_item from idlelib import iomenu # encoding @@ -462,7 +462,7 @@ def __init__(self, rpchandler): self.rpchandler = rpchandler self.locals = __main__.__dict__ - self.calltip = calltips.CallTips() + self.calltip = calltip.Calltip() self.autocomplete = autocomplete.AutoComplete() def runcode(self, code): diff -Nru python3.7-3.7.0/Lib/idlelib/runscript.py python3.7-3.7.1/Lib/idlelib/runscript.py --- python3.7-3.7.0/Lib/idlelib/runscript.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/runscript.py 2018-10-20 06:04:19.000000000 +0000 @@ -193,3 +193,8 @@ # XXX This should really be a function of EditorWindow... tkMessageBox.showerror(title, message, parent=self.editwin.text) self.editwin.text.focus_set() + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_runscript', verbosity=2,) diff -Nru python3.7-3.7.0/Lib/idlelib/scrolledlist.py python3.7-3.7.1/Lib/idlelib/scrolledlist.py --- python3.7-3.7.0/Lib/idlelib/scrolledlist.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/scrolledlist.py 2018-10-20 06:04:19.000000000 +0000 @@ -142,6 +142,8 @@ scrolled_list.append("Item %02d" % i) if __name__ == '__main__': - # At the moment, test_scrolledlist merely creates instance, like htest. + from unittest import main + main('idlelib.idle_test.test_scrolledlist', verbosity=2,) + from idlelib.idle_test.htest import run run(_scrolled_list) diff -Nru python3.7-3.7.0/Lib/idlelib/searchbase.py python3.7-3.7.1/Lib/idlelib/searchbase.py --- python3.7-3.7.0/Lib/idlelib/searchbase.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/searchbase.py 2018-10-20 06:04:19.000000000 +0000 @@ -192,9 +192,10 @@ def default_command(self, dummy): pass + if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_searchbase', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_searchbase', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_searchbase) diff -Nru python3.7-3.7.0/Lib/idlelib/searchengine.py python3.7-3.7.1/Lib/idlelib/searchengine.py --- python3.7-3.7.0/Lib/idlelib/searchengine.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/searchengine.py 2018-10-20 06:04:19.000000000 +0000 @@ -231,6 +231,7 @@ line, col = map(int, index.split(".")) # Fails on invalid index return line, col + if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_searchengine', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_searchengine', verbosity=2) diff -Nru python3.7-3.7.0/Lib/idlelib/search.py python3.7-3.7.1/Lib/idlelib/search.py --- python3.7-3.7.0/Lib/idlelib/search.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/search.py 2018-10-20 06:04:19.000000000 +0000 @@ -94,9 +94,8 @@ button.pack() if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_search', - verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_search', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_search_dialog) diff -Nru python3.7-3.7.0/Lib/idlelib/squeezer.py python3.7-3.7.1/Lib/idlelib/squeezer.py --- python3.7-3.7.0/Lib/idlelib/squeezer.py 1970-01-01 00:00:00.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/squeezer.py 2018-10-20 06:04:19.000000000 +0000 @@ -0,0 +1,355 @@ +"""An IDLE extension to avoid having very long texts printed in the shell. + +A common problem in IDLE's interactive shell is printing of large amounts of +text into the shell. This makes looking at the previous history difficult. +Worse, this can cause IDLE to become very slow, even to the point of being +completely unusable. + +This extension will automatically replace long texts with a small button. +Double-cliking this button will remove it and insert the original text instead. +Middle-clicking will copy the text to the clipboard. Right-clicking will open +the text in a separate viewing window. + +Additionally, any output can be manually "squeezed" by the user. This includes +output written to the standard error stream ("stderr"), such as exception +messages and their tracebacks. +""" +import re + +import tkinter as tk +from tkinter.font import Font +import tkinter.messagebox as tkMessageBox + +from idlelib.config import idleConf +from idlelib.textview import view_text +from idlelib.tooltip import Hovertip +from idlelib import macosx + + +def count_lines_with_wrapping(s, linewidth=80, tabwidth=8): + """Count the number of lines in a given string. + + Lines are counted as if the string was wrapped so that lines are never over + linewidth characters long. + + Tabs are considered tabwidth characters long. + """ + pos = 0 + linecount = 1 + current_column = 0 + + for m in re.finditer(r"[\t\n]", s): + # process the normal chars up to tab or newline + numchars = m.start() - pos + pos += numchars + current_column += numchars + + # deal with tab or newline + if s[pos] == '\n': + linecount += 1 + current_column = 0 + else: + assert s[pos] == '\t' + current_column += tabwidth - (current_column % tabwidth) + + # if a tab passes the end of the line, consider the entire tab as + # being on the next line + if current_column > linewidth: + linecount += 1 + current_column = tabwidth + + pos += 1 # after the tab or newline + + # avoid divmod(-1, linewidth) + if current_column > 0: + # If the length was exactly linewidth, divmod would give (1,0), + # even though a new line hadn't yet been started. The same is true + # if length is any exact multiple of linewidth. Therefore, subtract + # 1 before doing divmod, and later add 1 to the column to + # compensate. + lines, column = divmod(current_column - 1, linewidth) + linecount += lines + current_column = column + 1 + + # process remaining chars (no more tabs or newlines) + current_column += len(s) - pos + # avoid divmod(-1, linewidth) + if current_column > 0: + linecount += (current_column - 1) // linewidth + else: + # the text ended with a newline; don't count an extra line after it + linecount -= 1 + + return linecount + + +class ExpandingButton(tk.Button): + """Class for the "squeezed" text buttons used by Squeezer + + These buttons are displayed inside a Tk Text widget in place of text. A + user can then use the button to replace it with the original text, copy + the original text to the clipboard or view the original text in a separate + window. + + Each button is tied to a Squeezer instance, and it knows to update the + Squeezer instance when it is expanded (and therefore removed). + """ + def __init__(self, s, tags, numoflines, squeezer): + self.s = s + self.tags = tags + self.numoflines = numoflines + self.squeezer = squeezer + self.editwin = editwin = squeezer.editwin + self.text = text = editwin.text + + # the base Text widget of the PyShell object, used to change text + # before the iomark + self.base_text = editwin.per.bottom + + button_text = "Squeezed text (%d lines)." % self.numoflines + tk.Button.__init__(self, text, text=button_text, + background="#FFFFC0", activebackground="#FFFFE0") + + button_tooltip_text = ( + "Double-click to expand, right-click for more options." + ) + Hovertip(self, button_tooltip_text, hover_delay=80) + + self.bind("", self.expand) + if macosx.isAquaTk(): + # AquaTk defines <2> as the right button, not <3>. + self.bind("", self.context_menu_event) + else: + self.bind("", self.context_menu_event) + self.selection_handle( + lambda offset, length: s[int(offset):int(offset) + int(length)]) + + self.is_dangerous = None + self.after_idle(self.set_is_dangerous) + + def set_is_dangerous(self): + dangerous_line_len = 50 * self.text.winfo_width() + self.is_dangerous = ( + self.numoflines > 1000 or + len(self.s) > 50000 or + any( + len(line_match.group(0)) >= dangerous_line_len + for line_match in re.finditer(r'[^\n]+', self.s) + ) + ) + + def expand(self, event=None): + """expand event handler + + This inserts the original text in place of the button in the Text + widget, removes the button and updates the Squeezer instance. + + If the original text is dangerously long, i.e. expanding it could + cause a performance degradation, ask the user for confirmation. + """ + if self.is_dangerous is None: + self.set_is_dangerous() + if self.is_dangerous: + confirm = tkMessageBox.askokcancel( + title="Expand huge output?", + message="\n\n".join([ + "The squeezed output is very long: %d lines, %d chars.", + "Expanding it could make IDLE slow or unresponsive.", + "It is recommended to view or copy the output instead.", + "Really expand?" + ]) % (self.numoflines, len(self.s)), + default=tkMessageBox.CANCEL, + parent=self.text) + if not confirm: + return "break" + + self.base_text.insert(self.text.index(self), self.s, self.tags) + self.base_text.delete(self) + self.squeezer.expandingbuttons.remove(self) + + def copy(self, event=None): + """copy event handler + + Copy the original text to the clipboard. + """ + self.clipboard_clear() + self.clipboard_append(self.s) + + def view(self, event=None): + """view event handler + + View the original text in a separate text viewer window. + """ + view_text(self.text, "Squeezed Output Viewer", self.s, + modal=False, wrap='none') + + rmenu_specs = ( + # item structure: (label, method_name) + ('copy', 'copy'), + ('view', 'view'), + ) + + def context_menu_event(self, event): + self.text.mark_set("insert", "@%d,%d" % (event.x, event.y)) + rmenu = tk.Menu(self.text, tearoff=0) + for label, method_name in self.rmenu_specs: + rmenu.add_command(label=label, command=getattr(self, method_name)) + rmenu.tk_popup(event.x_root, event.y_root) + return "break" + + +class Squeezer: + """Replace long outputs in the shell with a simple button. + + This avoids IDLE's shell slowing down considerably, and even becoming + completely unresponsive, when very long outputs are written. + """ + @classmethod + def reload(cls): + """Load class variables from config.""" + cls.auto_squeeze_min_lines = idleConf.GetOption( + "main", "PyShell", "auto-squeeze-min-lines", + type="int", default=50, + ) + + def __init__(self, editwin): + """Initialize settings for Squeezer. + + editwin is the shell's Editor window. + self.text is the editor window text widget. + self.base_test is the actual editor window Tk text widget, rather than + EditorWindow's wrapper. + self.expandingbuttons is the list of all buttons representing + "squeezed" output. + """ + self.editwin = editwin + self.text = text = editwin.text + + # Get the base Text widget of the PyShell object, used to change text + # before the iomark. PyShell deliberately disables changing text before + # the iomark via its 'text' attribute, which is actually a wrapper for + # the actual Text widget. Squeezer, however, needs to make such changes. + self.base_text = editwin.per.bottom + + self.expandingbuttons = [] + from idlelib.pyshell import PyShell # done here to avoid import cycle + if isinstance(editwin, PyShell): + # If we get a PyShell instance, replace its write method with a + # wrapper, which inserts an ExpandingButton instead of a long text. + def mywrite(s, tags=(), write=editwin.write): + # only auto-squeeze text which has just the "stdout" tag + if tags != "stdout": + return write(s, tags) + + # only auto-squeeze text with at least the minimum + # configured number of lines + numoflines = self.count_lines(s) + if numoflines < self.auto_squeeze_min_lines: + return write(s, tags) + + # create an ExpandingButton instance + expandingbutton = ExpandingButton(s, tags, numoflines, + self) + + # insert the ExpandingButton into the Text widget + text.mark_gravity("iomark", tk.RIGHT) + text.window_create("iomark", window=expandingbutton, + padx=3, pady=5) + text.see("iomark") + text.update() + text.mark_gravity("iomark", tk.LEFT) + + # add the ExpandingButton to the Squeezer's list + self.expandingbuttons.append(expandingbutton) + + editwin.write = mywrite + + def count_lines(self, s): + """Count the number of lines in a given text. + + Before calculation, the tab width and line length of the text are + fetched, so that up-to-date values are used. + + Lines are counted as if the string was wrapped so that lines are never + over linewidth characters long. + + Tabs are considered tabwidth characters long. + """ + # Tab width is configurable + tabwidth = self.editwin.get_tk_tabwidth() + + # Get the Text widget's size + linewidth = self.editwin.text.winfo_width() + # Deduct the border and padding + linewidth -= 2*sum([int(self.editwin.text.cget(opt)) + for opt in ('border', 'padx')]) + + # Get the Text widget's font + font = Font(self.editwin.text, name=self.editwin.text.cget('font')) + # Divide the size of the Text widget by the font's width. + # According to Tk8.5 docs, the Text widget's width is set + # according to the width of its font's '0' (zero) character, + # so we will use this as an approximation. + # see: http://www.tcl.tk/man/tcl8.5/TkCmd/text.htm#M-width + linewidth //= font.measure('0') + + return count_lines_with_wrapping(s, linewidth, tabwidth) + + def squeeze_current_text_event(self, event): + """squeeze-current-text event handler + + Squeeze the block of text inside which contains the "insert" cursor. + + If the insert cursor is not in a squeezable block of text, give the + user a small warning and do nothing. + """ + # set tag_name to the first valid tag found on the "insert" cursor + tag_names = self.text.tag_names(tk.INSERT) + for tag_name in ("stdout", "stderr"): + if tag_name in tag_names: + break + else: + # the insert cursor doesn't have a "stdout" or "stderr" tag + self.text.bell() + return "break" + + # find the range to squeeze + start, end = self.text.tag_prevrange(tag_name, tk.INSERT + "+1c") + s = self.text.get(start, end) + + # if the last char is a newline, remove it from the range + if len(s) > 0 and s[-1] == '\n': + end = self.text.index("%s-1c" % end) + s = s[:-1] + + # delete the text + self.base_text.delete(start, end) + + # prepare an ExpandingButton + numoflines = self.count_lines(s) + expandingbutton = ExpandingButton(s, tag_name, numoflines, self) + + # insert the ExpandingButton to the Text + self.text.window_create(start, window=expandingbutton, + padx=3, pady=5) + + # insert the ExpandingButton to the list of ExpandingButtons, while + # keeping the list ordered according to the position of the buttons in + # the Text widget + i = len(self.expandingbuttons) + while i > 0 and self.text.compare(self.expandingbuttons[i-1], + ">", expandingbutton): + i -= 1 + self.expandingbuttons.insert(i, expandingbutton) + + return "break" + + +Squeezer.reload() + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_squeezer', verbosity=2, exit=False) + + # Add htest. diff -Nru python3.7-3.7.0/Lib/idlelib/stackviewer.py python3.7-3.7.1/Lib/idlelib/stackviewer.py --- python3.7-3.7.0/Lib/idlelib/stackviewer.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/stackviewer.py 2018-10-20 06:04:19.000000000 +0000 @@ -8,6 +8,7 @@ from idlelib.tree import TreeNode, TreeItem, ScrolledCanvas def StackBrowser(root, flist=None, tb=None, top=None): + global sc, item, node # For testing. if top is None: top = tk.Toplevel(root) sc = ScrolledCanvas(top, bg="white", highlightthickness=0) @@ -134,7 +135,6 @@ intentional_name_error except NameError: exc_type, exc_value, exc_tb = sys.exc_info() - # inject stack trace to sys sys.last_type = exc_type sys.last_value = exc_value @@ -148,5 +148,8 @@ del sys.last_traceback if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_stackviewer', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_stack_viewer) diff -Nru python3.7-3.7.0/Lib/idlelib/statusbar.py python3.7-3.7.1/Lib/idlelib/statusbar.py --- python3.7-3.7.0/Lib/idlelib/statusbar.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/statusbar.py 2018-10-20 06:04:19.000000000 +0000 @@ -42,5 +42,8 @@ frame.pack() if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_statusbar', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_multistatus_bar) diff -Nru python3.7-3.7.0/Lib/idlelib/textview.py python3.7-3.7.1/Lib/idlelib/textview.py --- python3.7-3.7.0/Lib/idlelib/textview.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/textview.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,15 +1,37 @@ """Simple text browser for IDLE """ -from tkinter import Toplevel, Text +from tkinter import Toplevel, Text, TclError,\ + HORIZONTAL, VERTICAL, N, S, E, W from tkinter.ttk import Frame, Scrollbar, Button from tkinter.messagebox import showerror +from idlelib.colorizer import color_config + + +class AutoHiddenScrollbar(Scrollbar): + """A scrollbar that is automatically hidden when not needed. + + Only the grid geometry manager is supported. + """ + def set(self, lo, hi): + if float(lo) > 0.0 or float(hi) < 1.0: + self.grid() + else: + self.grid_remove() + super().set(lo, hi) + + def pack(self, **kwargs): + raise TclError(f'{self.__class__.__name__} does not support "pack"') + + def place(self, **kwargs): + raise TclError(f'{self.__class__.__name__} does not support "place"') + class TextFrame(Frame): "Display text with scrollbar." - def __init__(self, parent, rawtext): + def __init__(self, parent, rawtext, wrap='word'): """Create a frame for Textview. parent - parent widget for this frame @@ -18,31 +40,40 @@ super().__init__(parent) self['relief'] = 'sunken' self['height'] = 700 - # TODO: get fg/bg from theme. - self.bg = '#ffffff' - self.fg = '#000000' - - self.text = text = Text(self, wrap='word', highlightthickness=0, - fg=self.fg, bg=self.bg) - self.scroll = scroll = Scrollbar(self, orient='vertical', - takefocus=False, command=text.yview) - text['yscrollcommand'] = scroll.set + + self.text = text = Text(self, wrap=wrap, highlightthickness=0) + color_config(text) + text.grid(row=0, column=0, sticky=N+S+E+W) + self.grid_rowconfigure(0, weight=1) + self.grid_columnconfigure(0, weight=1) text.insert(0.0, rawtext) text['state'] = 'disabled' text.focus_set() - scroll.pack(side='right', fill='y') - text.pack(side='left', expand=True, fill='both') + # vertical scrollbar + self.yscroll = yscroll = AutoHiddenScrollbar(self, orient=VERTICAL, + takefocus=False, + command=text.yview) + text['yscrollcommand'] = yscroll.set + yscroll.grid(row=0, column=1, sticky=N+S) + + if wrap == 'none': + # horizontal scrollbar + self.xscroll = xscroll = AutoHiddenScrollbar(self, orient=HORIZONTAL, + takefocus=False, + command=text.xview) + text['xscrollcommand'] = xscroll.set + xscroll.grid(row=1, column=0, sticky=E+W) class ViewFrame(Frame): "Display TextFrame and Close button." - def __init__(self, parent, text): + def __init__(self, parent, text, wrap='word'): super().__init__(parent) self.parent = parent self.bind('', self.ok) self.bind('', self.ok) - self.textframe = TextFrame(self, text) + self.textframe = TextFrame(self, text, wrap=wrap) self.button_ok = button_ok = Button( self, text='Close', command=self.ok, takefocus=False) self.textframe.pack(side='top', expand=True, fill='both') @@ -56,7 +87,7 @@ class ViewWindow(Toplevel): "A simple text viewer dialog for IDLE." - def __init__(self, parent, title, text, modal=True, + def __init__(self, parent, title, text, modal=True, wrap='word', *, _htest=False, _utest=False): """Show the given text in a scrollable window with a 'close' button. @@ -66,6 +97,7 @@ parent - parent of this dialog title - string which is title of popup dialog text - text to display in dialog + wrap - type of text wrapping to use ('word', 'char' or 'none') _htest - bool; change box location when running htest. _utest - bool; don't wait_window when running unittest. """ @@ -77,13 +109,14 @@ self.geometry(f'=750x500+{x}+{y}') self.title(title) - self.viewframe = ViewFrame(self, text) + self.viewframe = ViewFrame(self, text, wrap=wrap) self.protocol("WM_DELETE_WINDOW", self.ok) self.button_ok = button_ok = Button(self, text='Close', command=self.ok, takefocus=False) self.viewframe.pack(side='top', expand=True, fill='both') - if modal: + self.is_modal = modal + if self.is_modal: self.transient(parent) self.grab_set() if not _utest: @@ -91,23 +124,27 @@ def ok(self, event=None): """Dismiss text viewer dialog.""" + if self.is_modal: + self.grab_release() self.destroy() -def view_text(parent, title, text, modal=True, _utest=False): +def view_text(parent, title, text, modal=True, wrap='word', _utest=False): """Create text viewer for given text. parent - parent of this dialog title - string which is the title of popup dialog text - text to display in this dialog + wrap - type of text wrapping to use ('word', 'char' or 'none') modal - controls if users can interact with other windows while this dialog is displayed _utest - bool; controls wait_window on unittest """ - return ViewWindow(parent, title, text, modal, _utest=_utest) + return ViewWindow(parent, title, text, modal, wrap=wrap, _utest=_utest) -def view_file(parent, title, filename, encoding, modal=True, _utest=False): +def view_file(parent, title, filename, encoding, modal=True, wrap='word', + _utest=False): """Create text viewer for text in filename. Return error message if file cannot be read. Otherwise calls view_text @@ -125,12 +162,14 @@ message=str(err), parent=parent) else: - return view_text(parent, title, contents, modal, _utest=_utest) + return view_text(parent, title, contents, modal, wrap=wrap, + _utest=_utest) return None if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_textview', verbosity=2, exit=False) + from unittest import main + main('idlelib.idle_test.test_textview', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(ViewWindow) diff -Nru python3.7-3.7.0/Lib/idlelib/tooltip.py python3.7-3.7.1/Lib/idlelib/tooltip.py --- python3.7-3.7.0/Lib/idlelib/tooltip.py 2018-06-27 03:07:35.000000000 +0000 +++ python3.7-3.7.1/Lib/idlelib/tooltip.py 2018-10-20 06:04:19.000000000 +0000 @@ -1,80 +1,167 @@ -# general purpose 'tooltip' routines - currently unused in idlelib -# (although the 'calltips' extension is partly based on this code) -# may be useful for some purposes in (or almost in ;) the current project scope -# Ideas gleaned from PySol +"""Tools for displaying tool-tips. +This includes: + * an abstract base-class for different kinds of tooltips + * a simple text-only Tooltip class +""" from tkinter import * -class ToolTipBase: - def __init__(self, button): - self.button = button - self.tipwindow = None - self.id = None - self.x = self.y = 0 - self._id1 = self.button.bind("", self.enter) - self._id2 = self.button.bind("", self.leave) - self._id3 = self.button.bind("", self.leave) +class TooltipBase(object): + """abstract base class for tooltips""" - def enter(self, event=None): - self.schedule() + def __init__(self, anchor_widget): + """Create a tooltip. - def leave(self, event=None): - self.unschedule() - self.hidetip() + anchor_widget: the widget next to which the tooltip will be shown - def schedule(self): - self.unschedule() - self.id = self.button.after(1500, self.showtip) + Note that a widget will only be shown when showtip() is called. + """ + self.anchor_widget = anchor_widget + self.tipwindow = None - def unschedule(self): - id = self.id - self.id = None - if id: - self.button.after_cancel(id) + def __del__(self): + self.hidetip() def showtip(self): + """display the tooltip""" if self.tipwindow: return - # The tip window must be completely outside the button; + self.tipwindow = tw = Toplevel(self.anchor_widget) + # show no border on the top level window + tw.wm_overrideredirect(1) + try: + # This command is only needed and available on Tk >= 8.4.0 for OSX. + # Without it, call tips intrude on the typing process by grabbing + # the focus. + tw.tk.call("::tk::unsupported::MacWindowStyle", "style", tw._w, + "help", "noActivates") + except TclError: + pass + + self.position_window() + self.showcontents() + self.tipwindow.update_idletasks() # Needed on MacOS -- see #34275. + self.tipwindow.lift() # work around bug in Tk 8.5.18+ (issue #24570) + + def position_window(self): + """(re)-set the tooltip's screen position""" + x, y = self.get_position() + root_x = self.anchor_widget.winfo_rootx() + x + root_y = self.anchor_widget.winfo_rooty() + y + self.tipwindow.wm_geometry("+%d+%d" % (root_x, root_y)) + + def get_position(self): + """choose a screen position for the tooltip""" + # The tip window must be completely outside the anchor widget; # otherwise when the mouse enters the tip window we get # a leave event and it disappears, and then we get an enter # event and it reappears, and so on forever :-( - x = self.button.winfo_rootx() + 20 - y = self.button.winfo_rooty() + self.button.winfo_height() + 1 - self.tipwindow = tw = Toplevel(self.button) - tw.wm_overrideredirect(1) - tw.wm_geometry("+%d+%d" % (x, y)) - self.showcontents() + # + # Note: This is a simplistic implementation; sub-classes will likely + # want to override this. + return 20, self.anchor_widget.winfo_height() + 1 - def showcontents(self, text="Your text here"): - # Override this in derived class - label = Label(self.tipwindow, text=text, justify=LEFT, - background="#ffffe0", relief=SOLID, borderwidth=1) - label.pack() + def showcontents(self): + """content display hook for sub-classes""" + # See ToolTip for an example + raise NotImplementedError def hidetip(self): + """hide the tooltip""" + # Note: This is called by __del__, so careful when overriding/extending tw = self.tipwindow self.tipwindow = None if tw: - tw.destroy() + try: + tw.destroy() + except TclError: + pass + + +class OnHoverTooltipBase(TooltipBase): + """abstract base class for tooltips, with delayed on-hover display""" + + def __init__(self, anchor_widget, hover_delay=1000): + """Create a tooltip with a mouse hover delay. + + anchor_widget: the widget next to which the tooltip will be shown + hover_delay: time to delay before showing the tooltip, in milliseconds + + Note that a widget will only be shown when showtip() is called, + e.g. after hovering over the anchor widget with the mouse for enough + time. + """ + super(OnHoverTooltipBase, self).__init__(anchor_widget) + self.hover_delay = hover_delay + + self._after_id = None + self._id1 = self.anchor_widget.bind("", self._show_event) + self._id2 = self.anchor_widget.bind("", self._hide_event) + self._id3 = self.anchor_widget.bind("