diff -Nru r10k-3.7.0/.github/dependabot.yml r10k-4.0.0/.github/dependabot.yml --- r10k-3.7.0/.github/dependabot.yml 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/.github/dependabot.yml 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,17 @@ +version: 2 +updates: +# raise PRs for gem updates +- package-ecosystem: bundler + directory: "/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 + +# Maintain dependencies for GitHub Actions +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 diff -Nru r10k-3.7.0/.github/pull_request_template.md r10k-4.0.0/.github/pull_request_template.md --- r10k-3.7.0/.github/pull_request_template.md 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/.github/pull_request_template.md 2023-08-01 15:06:51.000000000 +0000 @@ -1,4 +1,4 @@ Please add all notable changes to the "Unreleased" section of the CHANGELOG in the format: ``` -- Summary of changes. [Issue or PR #](link to issue or PR) +- (JIRA ticket) Summary of changes. [Issue or PR #](link to issue or PR) ``` diff -Nru r10k-3.7.0/.github/workflows/docker.yml r10k-4.0.0/.github/workflows/docker.yml --- r10k-3.7.0/.github/workflows/docker.yml 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/.github/workflows/docker.yml 2023-08-01 15:06:51.000000000 +0000 @@ -3,7 +3,7 @@ on: push: branches: - - master + - main jobs: build-and-publish: @@ -17,7 +17,7 @@ runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - uses: azure/docker-login@v1 with: # This doesn't seem to work unless we point directly to the secrets username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -27,9 +27,17 @@ with: ruby-version: 2.6.x - run: gem install bundler + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Build container working-directory: docker - run: make lint build test + run: | + docker system prune --all --force --volumes + docker builder prune --force --keep-storage=10GB + make lint build test - name: Publish container working-directory: docker run: | diff -Nru r10k-3.7.0/.github/workflows/release.yml r10k-4.0.0/.github/workflows/release.yml --- r10k-3.7.0/.github/workflows/release.yml 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/.github/workflows/release.yml 2023-08-01 15:06:51.000000000 +0000 @@ -3,7 +3,7 @@ on: push: branches: - - master + - main paths: - 'lib/r10k/version.rb' @@ -11,14 +11,15 @@ release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: '0' - name: Bump version and push tag - uses: anothrNick/github-tag-action@1.17.2 + uses: anothrNick/github-tag-action@1.67.0 env: GITHUB_TOKEN: ${{ secrets.PUPPET_RELEASE_GH_TOKEN }} DEFAULT_BUMP: patch + TAG_CONTEXT: branch WITH_V: false # Uncomment this if the tag and version file become out-of-sync and # you need to tag at a specific version. diff -Nru r10k-3.7.0/.github/workflows/rspec_tests.yml r10k-4.0.0/.github/workflows/rspec_tests.yml --- r10k-3.7.0/.github/workflows/rspec_tests.yml 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/.github/workflows/rspec_tests.yml 2023-08-01 15:06:51.000000000 +0000 @@ -3,7 +3,8 @@ on: pull_request: branches: - - master + - main + - 3.x jobs: rspec_tests: @@ -11,19 +12,19 @@ strategy: matrix: cfg: - - {os: ubuntu-18.04, ruby: 2.4} - - {os: ubuntu-18.04, ruby: 2.5} - - {os: ubuntu-18.04, ruby: 2.6} - - {os: ubuntu-18.04, ruby: 2.7} - - {os: ubuntu-18.04, ruby: jruby-9.2.9.0} - - {os: windows-2016, ruby: 2.5} - - {os: windows-2016, ruby: 2.6} - - {os: windows-2016, ruby: 2.7} + - {os: ubuntu-latest, ruby: 2.6} + - {os: ubuntu-latest, ruby: 2.7} + - {os: ubuntu-latest, ruby: 3.1} + - {os: ubuntu-latest, ruby: 3.2} + - {os: ubuntu-latest, ruby: jruby-9.3} + - {os: ubuntu-latest, ruby: jruby-9.4} + - {os: windows-latest, ruby: 2.6} + - {os: windows-latest, ruby: 3.2} runs-on: ${{ matrix.cfg.os }} steps: - name: Checkout current PR - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install ruby version ${{ matrix.cfg.ruby }} uses: ruby/setup-ruby@v1 @@ -32,7 +33,6 @@ - name: Install bundler and gems run: | - gem install bundler bundle config set without packaging documentation bundle install --jobs 4 --retry 3 @@ -59,7 +59,7 @@ bundle --version # Run tests - bundle exec rspec --color --format documentation spec/unit + bundle exec rspec --color --format documentation spec - name: Run tests on Linux if: runner.os == 'Linux' @@ -78,4 +78,4 @@ fi # Run tests - bundle exec rspec --color --format documentation spec/unit + bundle exec rspec --color --format documentation spec diff -Nru r10k-3.7.0/.github/workflows/stale.yml r10k-4.0.0/.github/workflows/stale.yml --- r10k-3.7.0/.github/workflows/stale.yml 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/.github/workflows/stale.yml 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,21 @@ +name: Mark stale issues + +on: + schedule: + - cron: "30 1 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 60 + days-before-close: 7 + stale-issue-message: 'This issue has been marked stale because it has had no activity for 60 days. The Puppet Team is actively prioritizing existing bugs and new features, if this issue is still important to you please comment and we will add this to our backlog to complete. Otherwise, it will be closed in 7 days.' + stale-issue-label: 'stale' + exempt-issue-labels: 'community interest' + stale-pr-message: "This PR has been marked stale because it has had no activity for 60 days. If you are still interested in getting this merged, please comment and we'll try to move it forward. Otherwise, it will be closed in 7 days." + stale-pr-label: 'stale' + exempt-pr-labels: 'community interest' diff -Nru r10k-3.7.0/.gitignore r10k-4.0.0/.gitignore --- r10k-3.7.0/.gitignore 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/.gitignore 2023-08-01 15:06:51.000000000 +0000 @@ -7,3 +7,4 @@ integration/log integration/junit integration/configs +r10k.log diff -Nru r10k-3.7.0/.travis.yml r10k-4.0.0/.travis.yml --- r10k-3.7.0/.travis.yml 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,45 +0,0 @@ ---- -language: ruby -bundler_args: "--without system" -script: "bundle exec rspec --color --format documentation spec/unit" -notifications: - email: false -sudo: false -jdk: - - openjdk11 -before_install: gem install bundler -v '< 2' --no-document -matrix: - include: - - stage: r10k tests - rvm: 2.7.0 - - stage: r10k tests - rvm: 2.6.5 - - stage: r10k tests - rvm: 2.5.0 - - stage: r10k tests - rvm: 2.4.0 - - stage: r10k tests - rvm: 2.3.0 - - stage: r10k tests - rvm: jruby - - stage: r10k container tests - dist: focal - language: ruby - services: - - docker - rvm: 2.6.6 - env: - - DOCKER_COMPOSE_VERSION=1.25.5 - # necessary to prevent overwhelming TravisCI build output limits - - DOCKER_BUILD_FLAGS="--progress plain" - before_install: - - sudo rm /usr/local/bin/docker-compose - - curl --location https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname --kernel-name`-`uname --machine` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - script: - - set -e - - cd docker - - make lint - - make build - - make test diff -Nru r10k-3.7.0/CHANGELOG.mkd r10k-4.0.0/CHANGELOG.mkd --- r10k-3.7.0/CHANGELOG.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/CHANGELOG.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -4,6 +4,149 @@ Unreleased ---------- + +4.0.0 +----- +- Drop Ruby 2.3/2.4/2.5 support; use puppet_forge 4.1 or newer [#1336](https://github.com/puppetlabs/r10k/pull/1336) +- (maint) Add Ruby 3.0 to rspec CI matrix [#1261](https://github.com/puppetlabs/r10k/pull/1261) +- (RK-368) remove `purge_whitelist` setting [#1277](https://github.com/puppetlabs/r10k/pull/1277) +- (RK-390) Remove default ref for deploying git modules [#1275](http://github.com/puppetlabs/r10k/pull/1275) +- (RK-391) Change `exclude_spec` default to true for module spec dir deletion [#1264](https://github.com/puppetlabs/r10k/pull/1261) +- (RK-383) Remove deprecated `basedir` method from Puppetfile DSL. Users should use `environment_name` instead. [#1254](https://github.com/puppetlabs/r10k/pull/1254) +- (RK-386) Remove deprecated `bare` environment type. [#1235](https://github.com/puppetlabs/r10k/issues/1235) + + +3.16.0 +------ +- Emit more debug output when modules fail to sync [#1347](https://github.com/puppetlabs/r10k/pull/1347) +- Update GitHub Actions & introduce dependabot [#1337](https://github.com/puppetlabs/r10k/pull/1337) +- Update R10K proxy usage to follow newer rugged best practices [PE-35980](https://tickets.puppet.com/browse/PE-35980) +- Update Acceptance tests to be compatible with Puppet 8 [#1349](https://github.com/puppetlabs/r10k/pull/1349) + +3.15.4 +------ +- Pin dependencies to maintain support for old Ruby versions [#1329](https://github.com/puppetlabs/r10k/pull/1329) + +3.15.3 +------ +- Fix dirty working copy debug logging [#1321](https://github.com/puppetlabs/r10k/pull/1321) +- Allow gettext-setup < 2 for compatibility with Ruby 3.2 and Puppet 8 [#1325](https://github.com/puppetlabs/r10k/pull/1325) + +3.15.2 +------ +- Implement exclude regex for puppetfile install [#1248](https://github.com/puppetlabs/r10k/issues/1248) + +3.15.1 +------ +- Add TOC to configuration docs [#1298](https://github.com/puppetlabs/r10k/issues/1298) +- Remove the spec folder from gemspec [#1316](https://github.com/puppetlabs/r10k/issues/1316) + +3.15.0 +------ + +- Support and test Ruby 3 +- Allow puppet_forge 3.x & newer versions of fast_gettext/gettext [#1302](https://github.com/puppetlabs/r10k/pull/1302) +- Allow newer cri versions [#1302](https://github.com/puppetlabs/r10k/pull/1302) +- Fix error when using install_path from environment module [#1288](https://github.com/puppetlabs/r10k/issues/1288) +- (RK-399) Do not warn about local modifications in the spec directory when `exclude_spec` is set [#1291](https://github.com/puppetlabs/r10k/pull/1291) + +3.14.2 +------ + +- (RK-397) Ensure `--incremental` does not skip undeployed modules [#1278](https://github.com/puppetlabs/r10k/pull/1278) + +3.14.1 +------ + +- (RK-395) Make `exclude_spec` from a Puppetfile the priority override [#1271](https://github.com/puppetlabs/r10k/issues/1271) +- (RK-394) Fix `force` always resolving to true for `puppetfile install` [#1269](https://github.com/puppetlabs/r10k/issues/1265) +- (RK-393) Bug fix: not all spec directories are deleted when :exclude_spec is true [#1267](https://github.com/puppetlabs/r10k/pull/1267) +- Refactor internal module creation to always expect a hash, even for Forge modules, which can be specified in the Puppetfile with just a version string. [#1170](https://github.com/puppetlabs/r10k/pull/1170) + +3.14.0 +------ + +- Record unprocessed environment name, so that `strip_component` does not cause truncated environment names to be used as git branches, resulting in errors or incorrect deploys. [#1240](https://github.com/puppetlabs/r10k/pull/1240) +- (CODEMGMT-1294) Resync repos with unresolvable refs [#1239](https://github.com/puppetlabs/r10k/pull/1239) +- (RK-378) Restore access to the environment name from the Puppetfile [#1241](https://github.com/puppetlabs/r10k/pull/1241) +- (CODEMGMT-1300) Ensure the remote url in rugged cache directories is current [#1245](https://github.com/puppetlabs/r10k/pull/1245) +- Add support for tarball module type, allowing module content to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244) +- Add experimental support for tarball environment type, allowing whole environments to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244) + +3.13.0 +------ + +- Restore Ruby 3 compatibility [#1234](https://github.com/puppetlabs/r10k/pull/1234) +- (RK-381) Do not recurse into symlinked dirs when finding files to purge. [#1233](https://github.com/puppetlabs/r10k/pull/1233) +- Purge should remove unmanaged directories, in addition to unmanaged files. [#1222](https://github.com/puppetlabs/r10k/pull/1222) +- Rename experimental environment type "bare" to "plain". [#1228](https://github.com/puppetlabs/r10k/pull/1228) +- Add support for specifying additional logging ouputs. [#1230](https://github.com/puppetlabs/r10k/issues/1230) + +3.12.1 +------ + +- Fix requiring individual R10K::Actions without having already required 'r10k'. [#1223](https://github.com/puppetlabs/r10k/issues/1223) +- Fix evaluation of Puppetfiles that include local modules. [#1224](https://github.com/puppetlabs/r10k/pull/1224) + +3.12.0 +------ + +- (RK-308) Provide a `forge.allow_puppetfile_override` setting that, when true, causes a `forge` declaration in the Puppetfile to override `forge.baseurl`. [#1214](https://github.com/puppetlabs/r10k/pull/1214) +- (CODEMGMT-1415) Provide an `--incremental` flag to only sync those modules in a Puppetfile whose definitions have changed since last sync, or those whose versions could change. [#1200](https://github.com/puppetlabs/r10k/pull/1200) +- (CODEMGMT-1454) Ensure missing repo caches are re-synced [#1210](https://github.com/puppetlabs/r10k/pull/1210) +- (PF-2437) Allow token authentication to be used with the Forge. [#1192](https://github.com/puppetlabs/r10k/pull/1192) +- Only run the module postrun command for environments in which the module was modified. [#1215](https://github.com/puppetlabs/r10k/issues/1215) + +3.11.0 +------ + +- Always sync git cache on `ref: 'HEAD'` [#1182](https://github.com/puppetlabs/r10k/pull/1182) +- (CODEMGMT-1421, CODEMGMT-1422, CODEMGMT-1457) Add setting `exclude_spec` to remove the spec dir from module deployment[#1189](https://github.com/puppetlabs/r10k/pull/1189)[#1198](https://github.com/puppetlabs/r10k/pull/1198)[#1204](https://github.com/puppetlabs/r10k/pull/1204) +- (RK-369) Make module deploys run the postrun command if any environments were updated. [#982](https://github.com/puppetlabs/r10k/issues/982) +- Add support for Github App auth token. This allows r10k to authenticate under strict SSO/2FA guidelines that cannot utilize machine users for code deployment. [#1180](https://github.com/puppetlabs/r10k/pull/1180) +- Restore the ability to load a Puppetfile from a relative `basedir`. [#1202](https://github.com/puppetlabs/r10k/pull/1202), [#1203](https://github.com/puppetlabs/r10k/pull/1203) + +3.10.0 +------ + +- Add `authorization_token` setting to allow authentication to a custom Forge server. [#1181](https://github.com/puppetlabs/r10k/pull/1181) +- (RK-135) Attempting to download the latest version for a module that has no Forge releases will now issue a meaningful error. [#1177](https://github.com/puppetlabs/r10k/pull/1177) +- Added an interface to R10K::Source::Base named `reload!` for updating the environments list for a given deployment; `reload!` is called before deployment purges to make r10k deploy pools more threadsafe. [#1172](https://github.com/puppetlabs/r10k/pull/1172) +- Remove username and password from remote url in cache directory name [#1186](https://github.com/puppetlabs/r10k/pull/1186) +- Purging efficiency is greatly improved. R10K will no longer recurse into directories that match recursive purge exclusions. This should significantly improve the deploy times for those users who enable the "environment" purge level. [#1178](https://github.com/puppetlabs/r10k/pull/1178) + +3.9.3 +----- + +- Fixes a regression when using `--default_branch_override` with Puppetfiles containing Forge modules. [#1173](https://github.com/puppetlabs/r10k/issues/1173) + +3.9.2 +----- + +- Makes the third parameter to R10K::Actions optional, restoring backwards compatability broken in 3.9.1. + +3.9.1 +----- + +- Invalid module specifications in a Puppetfile will cause the R10K run to abort earlier than before. Prior to this release, the R10K run would complete, sync all other modules, and return an exit code of 1. R10K will now stop syncing modules and abort immediately. [#1161](https://github.com/puppetlabs/r10k/pull/1161) + +3.9.0 +----- + +- Add '--modules' flag to `deploy` subcommand as a replacement to '--puppetfile', deprecate '--puppetfile'. [#1147](https://github.com/puppetlabs/r10k/pull/1147) +- Deprecate 'purge_whitelist' and favor usage of 'purge_allowlist'. [#1144](https://github.com/puppetlabs/r10k/pull/1144) +- Add 'strip\_component' environment source configuration setting, to allow deploying Git branches named like "env/production" as Puppet environments named like "production". [#1128](https://github.com/puppetlabs/r10k/pull/1128) +- A warning will be emitted when the user supplies conflicting arguments to module definitions in a Puppetfile, such as when specifying both :commit and :branch [#1130](https://github.com/puppetlabs/r10k/pull/1130) +- Add optional standard module and environment specification interface: name, type, source, version. These options can be used when specifying environments and/or modules in a yaml/exec source, as well as when specifying modules in a Puppetfile. Providing the standard interface simplifies integrations with external services [#1131](https://github.com/puppetlabs/r10k/pull/1131) +- Pin cri to 2.15.10 to maintain support for Ruby 2.3 and 2.4 [#1121](https://github.com/puppetlabs/r10k/issues/1121) + +3.8.0 +----- + +- When a forge module fails name validation the offending name will now be printed in the error message. [#1126](https://github.com/puppetlabs/r10k/pull/1126) +- Module ref resolution will now fall back to the normal default branch if the default branch override cannot be resolved. [#1122](https://github.com/puppetlabs/r10k/pull/1122) +- Experimental feature change: conflicts between environment-defined modules and Puppetfile-defined modules now default to logging a warning and deploying the environment module version, overriding the Puppetfile. Previously, conflicts would result in an error. The behavior is now configurable via the `module_conflicts` environment setting [#1107](https://github.com/puppetlabs/r10k/pull/1107) + 3.7.0 ----- diff -Nru r10k-3.7.0/CODEOWNERS r10k-4.0.0/CODEOWNERS --- r10k-3.7.0/CODEOWNERS 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/CODEOWNERS 2023-08-01 15:06:51.000000000 +0000 @@ -1,2 +1,2 @@ -* @puppetlabs/puppetserver-maintainers @adrienthebo @dhollinger -/docker/ @puppetlabs/pupperware +# This repo is owned by the dumpling team +* @puppetlabs/dumpling diff -Nru r10k-3.7.0/Gemfile r10k-4.0.0/Gemfile --- r10k-3.7.0/Gemfile 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/Gemfile 2023-08-01 15:06:51.000000000 +0000 @@ -6,10 +6,10 @@ end group :development do - gem 'simplecov', '~> 0.9.1' + gem 'simplecov', '~> 0.22.0' gem 'ruby-prof', :platforms => :ruby end -if File.exists? "#{__FILE__}.local" +if File.exist? "#{__FILE__}.local" eval(File.read("#{__FILE__}.local"), binding) end diff -Nru r10k-3.7.0/README.mkd r10k-4.0.0/README.mkd --- r10k-3.7.0/README.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/README.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -5,6 +5,12 @@ [![Build Status](https://travis-ci.org/puppetlabs/r10k.png?branch=master)](https://travis-ci.org/puppetlabs/r10k) +> R10k is supported and maintained by Puppet, but we consider it to be feature +> complete and currently have no plans for any new development. We will keep it +> working within the context of Puppet Enterprise, but we cannot make any other +> maintenance promises at this time. + + Description ----------- @@ -14,17 +20,21 @@ modules. It implements the [Puppetfile](doc/puppetfile.mkd) format and provides a native implementation of Puppet [environments][workflow]. +You might also consider [g10k](https://github.com/xorpaul/g10k) as a non-ruby +based alternative. + + Requirements ------------ -R10k supports the Ruby versions `>= 2.4.0`. It's tested on Ruby 2.4.0 up to -Ruby 2.7.0 + Jruby. +R10k supports the Ruby versions `>= 2.6.0`. It's tested on Ruby 2.6.0 up to +Ruby 3.1.0 + Jruby. R10k requires additional components, depending on how you plan on managing environments and modules. - - Installing modules from the Puppet Forge requires Puppet 5.0.0+ or later. - Puppet 3 or 4 may work, but is generally not recommended. + - Installing modules from the Puppet Forge requires Puppet 7.0.0+ or later. + Puppet 5 and 6 may work, but is generally not recommended. - Git is required for creating environments and modules from Git - SVN is required for creating environments and modules from SVN @@ -61,11 +71,17 @@ If you have more specific needs or plan on modifying r10k you can run it out of a git repository using Bundler for dependencies: - git clone git://github.com/puppetlabs/r10k + git clone https://github.com/puppetlabs/r10k cd r10k bundle install bundle exec r10k help +### Arch Linux + +Arch Linux provides a [system package](https://archlinux.org/packages/community/any/r10k/) for r10k. +This is built against the [system Ruby](https://archlinux.org/packages/extra/x86_64/ruby/) (which is Ruby 3.0.2 as of 2021-08-03). +This package is maintained by [Tim Meusel](https://github.com/bastelfreak). + Usage ----- @@ -104,24 +120,13 @@ By default, a patch (Z) release will be triggered. To release a new major (X) or minor (Y) version, include `#major` or `#minor` (respectively) in your commit message to trigger the appropriate release. -NOTE: This currently only works for the default branch. If you would like to release from a different branch, please contact the [CODEOWNERS](CODEOWNERS). - Getting help ------------ - * IRC: r10k has a dedicated channel, `#r10k`, on Freenode where r10k questions - can be directed. Questions about r10k can frequently be asked in `#puppet` as well. + * [Puppet Community Slack](https://puppetcommunity.slack.com/) * Mailing lists: [puppet-users](https://groups.google.com/forum/#!forum/puppet-users) * Q&A: [Puppet Ask](https://ask.puppetlabs.com/questions/) -Contributors ------------- - -Please see the CHANGELOG for a listing of the (very awesome) contributors. - ## Maintenance -See [CODEOWNERS](CODEOWNERS) for active repo maintainers. - -Open [issues](https://github.com/puppetlabs/r10k/issues) directly in the r10k repo. - +See [CODEOWNERS](CODEOWNERS) for current project owners. diff -Nru r10k-3.7.0/azure-pipelines.yml r10k-4.0.0/azure-pipelines.yml --- r10k-3.7.0/azure-pipelines.yml 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/azure-pipelines.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,87 +0,0 @@ -# Don't run Azure when a branch is updated, only when a PR is updated. -# Prevents double builds when a PR is made from the main repo and not a fork. -trigger: none -pr: - autoCancel: true - branches: - include: - - '*' - -pool: - # self-hosted agent on Windows 10 1903 environment - # includes newer Docker engine with LCOW enabled, new build of LCOW image - # includes Ruby 2.5, Go 1.10, Node.js 10.10 - name: Default - -variables: - NAMESPACE: puppet - CONTAINER_NAME: r10k - CONTAINER_BUILD_PATH: . - LINT_IGNORES: - DOCKER_BUILDKIT: 1 - BUILD_OPTIONS: --build-arg alpine_version=3.9 - -workspace: - clean: resources - -steps: -- checkout: self # self represents the repo where the initial Pipelines YAML file was found - clean: true # whether to fetch clean each time - -- powershell: | - $gemfile = Join-Path -Path (Get-Location) -ChildPath 'docker/Gemfile' - $gempath = Join-Path -Path (Get-Location) -ChildPath 'docker/.bundle/gems' - bundle config --local gemfile $gemfile - bundle config --local path $gempath - bundle install --with test - displayName: Fetch Dependencies - timeoutInMinutes: 1 - name: fetch_deps - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Write-HostDiagnostics - displayName: Diagnostic Host Information - timeoutInMinutes: 2 - name: hostinfo - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Lint-Dockerfile -Name $ENV:CONTAINER_NAME -Ignore ($ENV:LINT_IGNORES -split ' ') - displayName: Lint $(CONTAINER_NAME) Dockerfile - timeoutInMinutes: 1 - name: lint_dockerfile - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Build-Container -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE -PathOrUri $ENV:CONTAINER_BUILD_PATH -AdditionalOptions ($ENV:BUILD_OPTIONS -split ' ') - displayName: Build $(CONTAINER_NAME) Container - timeoutInMinutes: 5 - name: build_container - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Initialize-TestEnv - displayName: Prepare Test Environment - name: test_prepare - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Invoke-ContainerTest -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE - displayName: Test $(CONTAINER_NAME) - timeoutInMinutes: 5 - name: test_container - -- task: PublishTestResults@2 - displayName: Publish $(CONTAINER_NAME) test results - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: 'docker/TEST-*.xml' - testRunTitle: $(CONTAINER_NAME) Test Results - -- powershell: | - . "$(bundle show pupperware)/ci/build.ps1" - Clear-BuildState -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE - displayName: Container Cleanup - timeoutInMinutes: 4 - condition: always() diff -Nru r10k-3.7.0/debian/changelog r10k-4.0.0/debian/changelog --- r10k-3.7.0/debian/changelog 2021-12-14 22:10:20.000000000 +0000 +++ r10k-4.0.0/debian/changelog 2024-02-23 02:53:48.000000000 +0000 @@ -1,3 +1,74 @@ +r10k (4.0.0-1ubuntu1) noble; urgency=medium + + * Merge with Debian unstable. Remaining changes: + - d/p/20_disable_test_changing_proxy_settings: skip tests trying to change + the proxy settings, they are failing in the Ubuntu autopkgtest + infrastructure (LP #1940104). + + -- Lucas Kanashiro Thu, 22 Feb 2024 23:53:48 -0300 + +r10k (4.0.0-1) unstable; urgency=medium + + * New upstream version 4.0.0 + * d/patches: refresh patchs + + -- Sebastien Badia Thu, 09 Nov 2023 01:13:24 +0100 + +r10k (3.15.4-1) unstable; urgency=medium + + [ Debian Janitor ] + * Remove constraints unnecessary since buster (oldstable) + * Remove obsolete field Name from debian/upstream/metadata + (already present in machine-readable debian/copyright). + Changes-By: lintian-brush + + [ Sebastien Badia ] + * New upstream version 3.15.4 + * Bump Standards-Version to 4.6.2 (no changes needed) + + -- Sebastien Badia Thu, 09 Feb 2023 16:44:35 +0100 + +r10k (3.15.2-2) unstable; urgency=medium + + * debian/control: + - Drop obsolete field XB-Ruby-Versions. + + -- Georg Faerber Mon, 24 Oct 2022 10:10:22 +0000 + +r10k (3.15.2-1) unstable; urgency=medium + + * New upstream version 3.15.2. + * debian/control: + - Bump Standards-Version to 4.6.1, no changes required. + * debian/patches: + - Refresh patch to relax versions of dependencies in gemspec. + + -- Georg Faerber Sun, 23 Oct 2022 16:14:16 +0000 + +r10k (3.14.2-2) unstable; urgency=medium + + * debian/patches: + - Relax dependencies in gemspec: cri, puppet_forge, fast_gettext, jwt. + - Skip another test which requires Internet access. + + -- Georg Faerber Fri, 15 Apr 2022 10:24:26 +0000 + +r10k (3.14.2-1) unstable; urgency=medium + + * New upstream version 3.14.2. + * debian/control: + - (Build-)Depend on ruby-jwt. + - Drop depending on ruby-interpreter, obsolete. + - Bump Standards-Version to 4.6.0, no changes required. + * debian/patches: + - Drop patch to ensure Ruby 3.0 compatibility, integrated upstream. + - Refresh remaining patches. + * debian/watch: + - Rely on 'special strings' provided by uscan to handle higher version + numbers. + + -- Georg Faerber Thu, 14 Apr 2022 19:00:38 +0000 + r10k (3.7.0-2.1ubuntu1) jammy; urgency=medium * Merge with Debian unstable. Remaining changes: diff -Nru r10k-3.7.0/debian/control r10k-4.0.0/debian/control --- r10k-3.7.0/debian/control 2021-12-14 22:10:20.000000000 +0000 +++ r10k-4.0.0/debian/control 2024-02-23 02:53:48.000000000 +0000 @@ -7,21 +7,22 @@ Markus Frosch , Georg Faerber , Build-Depends: debhelper-compat (= 13), - gem2deb (>= 0.6.1~), + gem2deb, git, rake, ronn | ruby-ronn (<< 0.7.3-5.1~), ruby-colored2, - ruby-cri (>= 2.6.1), - ruby-gettext-setup (>= 0.5), - ruby-log4r (>= 1.1.10), - ruby-minitar (>= 0.6.1), - ruby-multi-json (>= 1.10), + ruby-cri, + ruby-gettext-setup, + ruby-jwt (>= 2.2.3~), + ruby-log4r, + ruby-minitar, + ruby-multi-json, ruby-puppet-forge (>= 3.0.0~), - ruby-rspec (>= 3.1), - ruby-rugged (>= 0.24.0), - yard (>= 0.8.7.3), -Standards-Version: 4.5.1 + ruby-rspec, + ruby-rugged, + yard, +Standards-Version: 4.6.2 Vcs-Git: https://salsa.debian.org/puppet-team/r10k.git Vcs-Browser: https://salsa.debian.org/puppet-team/r10k Homepage: https://github.com/puppetlabs/r10k @@ -31,16 +32,16 @@ Package: r10k Architecture: all -XB-Ruby-Versions: ${ruby:Versions} -Depends: ruby | ruby-interpreter, +Depends: ruby, ruby-colored2, - ruby-cri (>= 2.6.1), - ruby-gettext-setup (>= 0.5), - ruby-log4r (>= 1.1.10), - ruby-minitar (>= 0.6.1), - ruby-multi-json (>= 1.10), + ruby-cri, + ruby-gettext-setup, + ruby-jwt (>= 2.2.3~), + ruby-log4r, + ruby-minitar, + ruby-multi-json, ruby-puppet-forge (>= 3.0.0~), - ruby-rugged (>= 0.24.0), + ruby-rugged, ${misc:Depends}, Recommends: git, Description: Puppet environment and module deployment diff -Nru r10k-3.7.0/debian/copyright r10k-4.0.0/debian/copyright --- r10k-3.7.0/debian/copyright 2021-11-26 23:24:35.000000000 +0000 +++ r10k-4.0.0/debian/copyright 2024-02-23 02:52:39.000000000 +0000 @@ -7,7 +7,7 @@ License: Apache-2.0 Files: debian/* -Copyright: 2014-2019 Sebastien Badia +Copyright: 2014-2023 Sebastien Badia License: Apache-2.0 Comment: the Debian packaging is licensed under the same terms as the original package. diff -Nru r10k-3.7.0/debian/patches/10-gemspec-relax-deps.patch r10k-4.0.0/debian/patches/10-gemspec-relax-deps.patch --- r10k-3.7.0/debian/patches/10-gemspec-relax-deps.patch 2021-11-26 23:24:35.000000000 +0000 +++ r10k-4.0.0/debian/patches/10-gemspec-relax-deps.patch 2024-02-23 02:52:39.000000000 +0000 @@ -1,25 +1,19 @@ -Description: gemspec: relax dep on fast_gettext +Description: gemspec: relax deps Author: Georg Faerber Forwarded: not-needed -Last-Update: 2020-03-10 +Last-Update: 2022-10-23 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ ---- a/r10k.gemspec -+++ b/r10k.gemspec -@@ -28,13 +28,13 @@ Gem::Specification.new do |s| - s.add_dependency 'log4r', '1.1.10' - s.add_dependency 'multi_json', '~> 1.10' +Index: r10k/r10k.gemspec +=================================================================== +--- r10k.orig/r10k.gemspec 2022-10-23 14:27:07.299637598 +0000 ++++ r10k/r10k.gemspec 2022-10-23 16:13:52.415575723 +0000 +@@ -34,7 +34,7 @@ + s.add_dependency 'fast_gettext', ['>= 1.1.0', '< 3.0.0'] + s.add_dependency 'gettext', ['>= 3.0.2', '< 4.0.0'] -- s.add_dependency 'puppet_forge', '~> 2.3.0' -+ s.add_dependency 'puppet_forge', '>= 3.0.0' - - s.add_dependency 'gettext-setup', '~>0.24' - # These two pins narrow what is allowed by gettext-setup, - # to preserver compatability with Ruby 2.4 -- s.add_dependency 'fast_gettext', '~> 1.1.0' -- s.add_dependency 'gettext', ['>= 3.0.2', '< 3.3.0'] -+ s.add_dependency 'fast_gettext', '>= 1.1.0' -+ s.add_dependency 'gettext', ['>= 3.0.2'] +- s.add_dependency 'jwt', '~> 2.2.3' ++ s.add_dependency 'jwt', '>= 2.2.3' + s.add_dependency 'minitar', '~> 0.9' s.add_development_dependency 'rspec', '~> 3.1' - diff -Nru r10k-3.7.0/debian/patches/12_disable_test_with_network_access r10k-4.0.0/debian/patches/12_disable_test_with_network_access --- r10k-3.7.0/debian/patches/12_disable_test_with_network_access 2021-11-26 23:24:35.000000000 +0000 +++ r10k-4.0.0/debian/patches/12_disable_test_with_network_access 2024-02-23 02:52:39.000000000 +0000 @@ -3,25 +3,25 @@ in order to fetch module version. Author: Sebastien Badia Forwarded: not-needed -Last-Update: 2020-02-07 +Last-Update: 2022-04-15 --- a/spec/unit/module/forge_spec.rb +++ b/spec/unit/module/forge_spec.rb -@@ -63,7 +63,7 @@ describe R10K::Module::Forge do +@@ -93,7 +93,7 @@ describe R10K::Module::Forge do context "when a module is deprecated" do - subject { described_class.new('puppetlabs/corosync', fixture_modulepath, :latest) } + subject { described_class.new('puppetlabs/corosync', fixture_modulepath, { version: :latest }) } - it "warns on sync if module is not already insync" do + xit "warns on sync if module is not already insync" do allow(subject).to receive(:status).and_return(:absent) allow(R10K::Forge::ModuleRelease).to receive(:new).and_return(double('mod_release', install: true)) -@@ -179,7 +179,7 @@ describe R10K::Module::Forge do +@@ -243,7 +243,7 @@ describe R10K::Module::Forge do end describe '#install' do - it 'installs the module from the forge' do + xit 'installs the module from the forge' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) release = instance_double('R10K::Forge::ModuleRelease') expect(R10K::Forge::ModuleRelease).to receive(:new).with('branan-eight_hundred', '8.0.0').and_return(release) diff -Nru r10k-3.7.0/debian/patches/13_fix_ruby_30_kwargs.patch r10k-4.0.0/debian/patches/13_fix_ruby_30_kwargs.patch --- r10k-3.7.0/debian/patches/13_fix_ruby_30_kwargs.patch 2021-11-26 23:24:35.000000000 +0000 +++ r10k-4.0.0/debian/patches/13_fix_ruby_30_kwargs.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,36 +0,0 @@ -From: Daniel Leidert -Date: Sat, 27 Nov 2021 00:20:48 +0100 -Subject: Fix Ruby 3.0 keyword arguments - -Bug-Debian: https://bugs.debian.org/996116 ---- - lib/r10k/git/rugged/bare_repository.rb | 2 +- - lib/r10k/git/rugged/working_repository.rb | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/lib/r10k/git/rugged/bare_repository.rb b/lib/r10k/git/rugged/bare_repository.rb -index 9dd25cc..b712be1 100644 ---- a/lib/r10k/git/rugged/bare_repository.rb -+++ b/lib/r10k/git/rugged/bare_repository.rb -@@ -64,7 +64,7 @@ class R10K::Git::Rugged::BareRepository < R10K::Git::Rugged::BaseRepository - results = nil - - R10K::Git.with_proxy(proxy) do -- results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } -+ results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } - end - - report_transfer(results, remote_name) -diff --git a/lib/r10k/git/rugged/working_repository.rb b/lib/r10k/git/rugged/working_repository.rb -index f78391f..c5a26fe 100644 ---- a/lib/r10k/git/rugged/working_repository.rb -+++ b/lib/r10k/git/rugged/working_repository.rb -@@ -93,7 +93,7 @@ class R10K::Git::Rugged::WorkingRepository < R10K::Git::Rugged::BaseRepository - results = nil - - R10K::Git.with_proxy(proxy) do -- results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } -+ results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } - end - - report_transfer(results, remote) diff -Nru r10k-3.7.0/debian/patches/series r10k-4.0.0/debian/patches/series --- r10k-3.7.0/debian/patches/series 2021-12-14 22:10:20.000000000 +0000 +++ r10k-4.0.0/debian/patches/series 2024-02-23 02:53:11.000000000 +0000 @@ -1,5 +1,4 @@ -10-gemspec-relax-deps.patch +#10-gemspec-relax-deps.patch 11_locales_path 12_disable_test_with_network_access -13_fix_ruby_30_kwargs.patch 20_disable_test_changing_proxy_settings diff -Nru r10k-3.7.0/debian/upstream/metadata r10k-4.0.0/debian/upstream/metadata --- r10k-3.7.0/debian/upstream/metadata 2021-11-26 23:24:35.000000000 +0000 +++ r10k-4.0.0/debian/upstream/metadata 2024-02-23 02:52:39.000000000 +0000 @@ -3,6 +3,5 @@ Bug-Database: https://github.com/puppetlabs/r10k/issues Bug-Submit: https://github.com/puppetlabs/r10k/issues Changelog: https://github.com/puppetlabs/r10k/tags -Name: r10k Repository: https://github.com/puppetlabs/r10k.git Repository-Browse: https://github.com/puppetlabs/r10k diff -Nru r10k-3.7.0/debian/watch r10k-4.0.0/debian/watch --- r10k-3.7.0/debian/watch 2021-11-26 23:24:35.000000000 +0000 +++ r10k-4.0.0/debian/watch 2024-02-23 02:52:39.000000000 +0000 @@ -1,2 +1,2 @@ version=4 -https://github.com/puppetlabs/r10k/tags .*/(\d\.\d\.\d)\.tar\.gz +https://github.com/puppetlabs/r10k/tags .*/@ANY_VERSION@@ARCHIVE_EXT@ diff -Nru r10k-3.7.0/doc/common-patterns.mkd r10k-4.0.0/doc/common-patterns.mkd --- r10k-3.7.0/doc/common-patterns.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/doc/common-patterns.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -12,8 +12,7 @@ to store your `Puppetfile`. Hiera data should be in the Control repo OR as a separate source in -`r10k.yaml`. Any `hiera.yaml` in the Control repo will be ignored on a per -environment basis, locating it at `/etc/puppetlabs/puppet/hiera.yaml` is prefered. +`r10k.yaml`. Each puppet module should be contained in its own independent forge module or repository. @@ -42,3 +41,4 @@ * [Reaktor](https://github.com/pzim/reaktor) * [zack/r10k's Webhooks](https://forge.puppetlabs.com/zack/r10k#webhook-support) (Puppet Enterprise only) +* [Simple Puppet Provisioner](https://github.com/mbaynton/SimplePuppetProvisioner) diff -Nru r10k-3.7.0/doc/dynamic-environments/configuration.mkd r10k-4.0.0/doc/dynamic-environments/configuration.mkd --- r10k-3.7.0/doc/dynamic-environments/configuration.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/doc/dynamic-environments/configuration.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -1,6 +1,70 @@ Dynamic Environment Configuration ================================= + + +* [Config file location](#config-file-location) + * [Manual configuration](#manual-configuration) + * [Automatic configuration](#automatic-configuration) +* [General options](#general-options) + * [cachedir](#cachedir) + * [proxy](#proxy) + * [pool_size](#pool_size) + * [git](#git) + * [provider](#provider) + * [proxy](#proxy-1) + * [username](#username) + * [private_key](#private_key) + * [oauth_token](#oauth_token) + * [repositories](#repositories) + * [private_key](#private_key-1) + * [oauth_token](#oauth_token-1) + * [proxy](#proxy-2) + * [forge](#forge) + * [proxy](#proxy-3) + * [baseurl](#baseurl) + * [authorization_token](#authorization_token) + * [allow_puppetfile_override](#allow_puppetfile_override) +* [Deployment options](#deployment-options) + * [postrun](#postrun) + * [sources](#sources) + * [deploy](#deploy) + * [purge\_levels](#purge_levels) + * [deployment](#deployment) + * [environment](#environment) + * [puppetfile](#puppetfile) + * [purge\_allowlist](#purge_allowlist) + * [write\_lock](#write_lock) + * [generate\_types](#generate_types) + * [puppet\_path](#puppet_path) + * [puppet\_conf](#puppet_conf) + * [exclude_spec](#exclude_spec) +* [Source options](#source-options) + * [remote](#remote) + * [basedir](#basedir) + * [prefix](#prefix) + * [prefix behaviour](#prefix-behaviour) + * [strip\_component](#strip_component) + * [strip\_component behaviour](#strip_component-behaviour) + * [ignore_branch_prefixes](#ignore_branch_prefixes) + * [ignore_branch_prefixes behaviour](#ignore_branch_prefixes-behaviour) + * [filter_command](#filter_command) +* [Examples](#examples) + * [Minimal example](#minimal-example) + * [Separate hiera data](#separate-hiera-data) + * [Multiple tenancy](#multiple-tenancy) + * [Multiple tenancy with external hieradata](#multiple-tenancy-with-external-hieradata) +* [Experimental Features](#experimental-features) + * [YAML Environment Source](#yaml-environment-source) + * [YAMLdir Environment Source](#yamldir-environment-source) + * [Exec environment Source](#exec-environment-source) + * [Environment Modules](#environment-modules) + * [Puppetfile module conflicts](#puppetfile-module-conflicts) + * [Plain Environment Type](#plain-environment-type) + * [Tarball Environment Type](#tarball-environment-type) + + + R10k uses a configuration file to determine how dynamic environments should be deployed. @@ -96,6 +160,18 @@ See the [git provider documentation](../git/providers.mkd) for more information regarding Git providers. +#### default_ref + +r10k is unable to deploy a git module if no `ref` is specified. A `default_ref` can be +set in the r10k config that will become the ref a module uses if not otherwise specified. This +is the lowest priority setting for a module's `ref`. Read the [Puppetfile documentation](../puppetfile.mkd#git) +for higher priority settings to determine a module's ref. + +```yaml +git: + default_ref: main +``` + #### proxy The 'proxy' setting allows you to set or override the global proxy setting specifically @@ -128,6 +204,18 @@ private_key: "/etc/puppetlabs/r10k/ssh/id_rsa" ``` +#### oauth_token + +The oauth_token setting is only used by the Rugged git provider. + +The oauth_token option specifies the path to the default access token for Git HTTPS remotes. +Public git repositories can be accessed via HTTPS without authentication, but the oauth_token setting may be set if any non-public HTTPS remotes are used. + +```yaml +git: + oauth_token: "/etc/puppetlabs/r10k/token" +``` + #### repositories The repositories option allows configuration to be set on a per-remote basis. Each entry is a map of @@ -145,6 +233,18 @@ private_key: "/etc/puppetlabs/r10k/ssh/id_rsa-protected-repo-deploy-key" ``` +##### oauth_token + +A repository specific access token to use for HTTPS connections for the given repository URL. This +overrides the global oauth_token setting. + +```yaml +git: + repositories: + - remote: "https://tessier-ashpool.freeside/protected-repo.git" + oauth_token: "/etc/puppetlabs/r10k/protected-repo-deploy-token" +``` + ##### proxy The 'proxy' setting allows you to set or override the global proxy setting for a single, specific @@ -165,11 +265,22 @@ The 'baseurl' setting indicates where Forge modules should be installed from. This defaults to 'https://forgeapi.puppetlabs.com' +#### authorization_token + +The 'authorization_token' setting allows you to provide a token for authenticating to a Forge server. +You will need to prepend your token with 'Bearer ' to authenticate to the Forge or when using your own Artifactory server. + ```yaml forge: baseurl: 'https://private-forge.mysite' + authorization_token: 'Bearer mysupersecretauthtoken' ``` +#### allow_puppetfile_override + +The `allow_puppetfile_override` setting causes r10k to respect [`forge` declarations](https://github.com/puppetlabs/r10k/blob/main/doc/puppetfile.mkd#forge) +in Puppetfiles, overriding the `baseurl` setting and allowing per-environment configuration of the Forge URL. + Deployment options ------------------ @@ -191,7 +302,7 @@ The postrun setting can only be set once. Occurrences of the string `$modifiedenvs` in the postrun command will be -replaced with the current environment(s) being deployed. +replaced with the current environment(s) being deployed, space separated. ### sources @@ -249,9 +360,9 @@ environment, nor declared in a Puppetfile committed to that branch. Enabling this purge level will cause r10k to load and parse the Puppetfile for -the environment even without the `--puppetfile` flag being set. However, +the environment even without the `--modules` flag being set. However, Puppetfile content will still only be deployed if the environment is new or -the `--puppetfile` flag is set. Additionally, no environment-level content +the `--modules` flag is set. Additionally, no environment-level content will be purged if any errors are encountered while evaluating the Puppetfile or deploying its contents. @@ -266,31 +377,31 @@ "modules") as well as alternate directories specified as an `install_path` option to any Puppetfile content declarations. -#### purge\_whitelist +#### purge\_allowlist -The `purge_whitelist` setting exempts the specified filename patterns from +The `purge_allowlist` setting exempts the specified filename patterns from being purged. This setting is currently only considered during `environment` level purging. (See above.) Given value must be a list of shell style filename patterns in string format. See the Ruby [documentation for the `fnmatch` method](http://ruby-doc.org/core-2.2.0/File.html#method-c-fnmatch) for more details on valid patterns. Note that the `FNM_PATHNAME` and -`FNM_DOTMATCH` flags are in effect when r10k considers the whitelist. +`FNM_DOTMATCH` flags are in effect when r10k considers the allowlist. Patterns are relative to the root of the environment being purged and *do -not match recursively* by default. For example, a whitelist value of +not match recursively* by default. For example, a allowlist value of `*myfile*` would only preserve a matching file at the root of the environment. To preserve the file throughout the deployed environment, a recursive pattern such as `**/*myfile*` would be required. -Files matching a whitelist pattern may still be removed if they exist in +Files matching a allowlist pattern may still be removed if they exist in a folder that is otherwise subject to purging. In this case, an additional -whitelist rule to preserve the containing folder is required. +allowlist rule to preserve the containing folder is required. ```yaml --- deploy: - purge_whitelist: [ 'custom.json', '**/*.xpp' ] + purge_allowlist: [ 'custom.json', '**/*.xpp' ] ``` @@ -336,6 +447,21 @@ puppet_conf: '/opt/puppet/conf/puppet.conf' ``` +#### exclude_spec + +During module deployment, r10k's default behavior is to delete the spec directory. Setting +`exclude_spec` to true will deploy modules without their spec directory. This behavior +can be configured for all modules using the `exclude_spec` setting in the r10k config. +It can also be passed as a CLI argument for `deploy environment/module`, overriding the +r10k config. Setting this per module in a `Puppetfile` will override the default, r10k config, +and cli flag for that module. The following example sets all modules to not deploy the spec +dir via the r10k config. + +```yaml +deploy: + exclude_spec: true +``` + Source options -------------- @@ -348,13 +474,16 @@ The 'remote' setting specifies where the source repository should be fetched from. It may be any valid URL that the source may check out or clone. The remote must be able to be fetched without any interactive input, eg usernames or -passwords cannot be prompted for in order to fetch the remote. +passwords cannot be prompted for in order to fetch the remote. We support the +`git`, `ssh`, and `https` transport protocols. An SSH private key or access +token must be provided for authentication. Only `https` may be used without +authentication. See [GitHub's blog on protocol security](https://github.blog/2021-09-01-improving-git-protocol-security-github/) for more info. ```yaml --- sources: mysource: - remote: 'git://git-server.site/my-org/main-modules' + remote: 'https://git-server.site/my-org/main-modules' ``` ### basedir @@ -394,6 +523,23 @@ * if `false` (default) environment folder will not be prefixed * if `String` environment folder will be prefixed with the `prefix` value. +### strip\_component + +The 'strip\_component' setting allows parts of environment names from a source to have a transformation applied, removing a part of the name before turning them into Puppet environments. This is primarily useful for VCS sources (e.g. Git), because it allows branch names to use prefixes or organizing name components such as "env/production", "env/development", but deploy Puppet environments from these branches named without the leading "env/" component. E.g. "production", "development". + +```yaml +--- +sources: + mysource: + basedir: '/etc/puppet/environments' + strip_component: 'env/' +``` + +#### strip\_component behaviour + +* if `string` environment names will have this prefix removed, if the prefix is present. Note that when string values are used, names can only have prefix components removed. +* if `/regex/` the regex will be matched against environment names and if a match is found, the matching name component will be removed. + ### ignore_branch_prefixes The 'ignore_branch_prefixes' setting causes environments to be ignored which match in part or whole @@ -455,7 +601,7 @@ --- sources: operations: - remote: 'git://git-server.site/my-org/org-modules' + remote: 'https://git-server.site/my-org/org-modules' basedir: '/etc/puppet/environments' ``` @@ -468,10 +614,10 @@ --- sources: operations: - remote: 'git://git-server.site/my-org/org-modules' + remote: 'https://git-server.site/my-org/org-modules' basedir: '/etc/puppet/environments' hiera: - remote: 'git://git-server.site/my-org/org-hiera-data' + remote: 'https://git-server.site/my-org/org-hiera-data' basedir: '/etc/puppet/hiera-data' ``` @@ -486,15 +632,15 @@ --- sources: main: - remote: 'git://git-server.site/my-org/main-modules' + remote: 'https://git-server.site/my-org/main-modules' basedir: '/etc/puppet/environments' prefix: false # Prefix defaults to false so this is only here for clarity qa: - remote: 'git://git-server.site/my-org/qa-puppet-modules' + remote: 'https://git-server.site/my-org/qa-puppet-modules' basedir: '/etc/puppet/environments' prefix: true dev: - remote: 'git://git-server.site/my-org/dev-puppet-modules' + remote: 'https://git-server.site/my-org/dev-puppet-modules' basedir: '/etc/puppet/environments' prefix: true ``` @@ -520,11 +666,11 @@ --- sources: app1_data: - remote: 'git://git-server.site/my-org/app1-hieradata' + remote: 'https://git-server.site/my-org/app1-hieradata' basedir: '/etc/puppet/hieradata' prefix: "app1" app1_modules: - remote: 'git://git-server.site/my-org/app1-puppet-modules' + remote: 'https://git-server.site/my-org/app1-puppet-modules' basedir: '/etc/puppet/environments' prefix: "app1" ``` @@ -569,13 +715,13 @@ --- production: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 development: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 ``` ### YAMLdir Environment Source @@ -606,8 +752,8 @@ # production.yaml --- type: git -remote: git@github.com:puppetlabs/control-repo.git -ref: 8820892 +source: git@github.com:puppetlabs/control-repo.git +version: 8820892 ``` ### Exec environment Source @@ -628,7 +774,7 @@ The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type. -For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile. +For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile. For ease of reading and consistency, however, it is perferred to use the generic type, source, and version options over implementation-specific formats and options such as "ref" and "git". The example below includes two Forge modules and one module sourced from a Git repository. The two environments are almost identical. However, a new version of the stdlib module has been deployed in development (6.2.0), that has not yet been deployed to production. @@ -636,25 +782,35 @@ --- production: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 development: type: git - remote: git@github.com:puppetlabs/control-repo.git - ref: 8820892 + source: git@github.com:puppetlabs/control-repo.git + version: 8820892 modules: - puppetlabs-stdlib: 6.2.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.2.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 ``` An example of a single environment definition for the YAMLdir environment source type: @@ -663,39 +819,108 @@ # production.yaml --- type: git -remote: git@github.com:puppetlabs/control-repo.git -ref: 8820892 +source: git@github.com:puppetlabs/control-repo.git +version: 8820892 +modules: + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 + reidmv-xampl: + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 +``` + +#### Puppetfile module conflicts + +When a module is defined in an environment and also in a Puppetfile, the default behavior is for the environment definition of the module to take precedence, a warning to be logged, and the Puppetfile definition to be ignored. The behavior is configurable to optionally skip the warning, or allow a hard failure instead. Use the `module_conflicts` option in an environment definition to control this. + +Available `module_conflicts` options: + +* `override_and_warn` (default): the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored. A warning will be printed. +* `override`: the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored. +* `error`: an error will be raised alerting the user to the conflict. The environment will not be deployed. + +```yaml +# production.yaml +--- +type: git +source: git@github.com:puppetlabs/control-repo.git +version: 8820892 +module_conflicts: override_and_warn modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 ``` -### Bare Environment Type +### Plain Environment Type A "control repository" typically contains a hiera.yaml, an environment.conf, a manifests/site.pp file, and a few other things. However, none of these are strictly necessary for an environment to be functional if modules can be deployed to it. -The bare environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly. +The plain environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly. ```yaml --- production: - type: bare + type: plain + modules: + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 + reidmv-xampl: + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 + +development: + type: plain modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 + puppetlabs-stdlib: + type: forge + version: 6.0.0 + puppetlabs-concat: + type: forge + version: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 +``` + +### Tarball Environment Type + +The tarball environment type allows an environment to be deployed from a tarball archive, rather than a Git repository. When using a tarball environment type, a source location for the tarball is required. Optionally, the tarball's sha256 checksum may be specified as the version. It is highly recommended to include a version specifier. If a version specifier is not included, r10k will never invalidate a cached copy of the tarball's source. + +Tarball environment sources will be unpacked directly into the environment root. + +```yaml +--- +production: + type: tarball + source: https://repo.example.com/projects/puppet/env-2.36.1.tar.gz + version: 99a906c99c2f144de43f2ae500509a7474ed11c583fb623efa8e5b377a3157f0 # sha256digest development: - type: bare + type: tarball + source: https://repo.example.com/projects/puppet/env-6128ada.tar.gz + version: 6128ada158622cd90f8e1360fb7c2c3830a812d1ec26ddf0db7eb16d61b7293f # sha256digest modules: - puppetlabs-stdlib: 6.0.0 - puppetlabs-concat: 6.1.0 reidmv-xampl: - git: https://github.com/reidmv/reidmv-xampl.git - ref: 62d07f2 + type: git + source: https://github.com/reidmv/reidmv-xampl.git + version: 62d07f2 ``` diff -Nru r10k-3.7.0/doc/dynamic-environments/usage.mkd r10k-4.0.0/doc/dynamic-environments/usage.mkd --- r10k-3.7.0/doc/dynamic-environments/usage.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/doc/dynamic-environments/usage.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -18,16 +18,16 @@ Recursively update all environments: - r10k deploy environment --puppetfile + r10k deploy environment --modules The simplest way to use r10k is by simply updating all environments and modules and takes the brute force approach of "update everything, ever." When this command is run r10k will update all sources, create new environments and delete old environments, and recursively update all environment modules specified in -environment Puppetfiles. While this is the simplest method for running r10k, is -is also the slowest by a very large degree because it does the maximum possible -work. This should not be something you run interactively, or use on a regular -basis. +environment Puppetfiles, yamldirs, etc. While this is the simplest method for +running r10k, it is also the slowest by a very large degree because it does the +maximum possible work. This should not be something you run interactively, or +use on a regular basis. - - - @@ -55,22 +55,53 @@ Update a single environment and force an update of modules: - r10k deploy environment my_working_environment --puppetfile + r10k deploy environment my_working_environment --modules This will update the given environment and update all contained modules. This is useful if you want to make sure that a given environment is fully up to date. - - - +There is also a middle ground between updating all modules and updating no modules. +It is often desirable to update the environment and then update only those modules +whose definitions have changed in the Puppetfile, or whose content _could_ have +changed since the last deployment (eg, Forge modules with their version set to +`:latest` or Git modules who point to a `branch` ref). + +This can be achieved by assuming content is unchanged locally on disk. This is the +opposite of what one would assume during a module development cycle, when a user +might be making local edits to test code changes. However, in production, access +to puppet code is usually locked down, and updates are deployed through automated +invocations of R10K. + +In these cases, deploys where most modules are unchanged and reference exact +versions (ie, not `:latest` or a branch as mentioned above), this invocation +may shorten deployment times dozens of seconds if not minutes depending on how +many modules meet the above criteria (approximately 1 minute for every 400 modules). + +To take advantage of this, set as many modules as possible in the Puppetfile to +explicit, static version. These are released Forge versions, or Git modules using +the `:tag`, or `:commit` keys. Git `:ref`s containing only the full 40 character +commit SHA will also be treated as static versions. Then invoke a deploy with: + +There may be issues with deployments apparently successful after an initial errored +deployment. If this is happening, try running without the `--incremental` flag +to run a full deployment. + + r10k deploy environment production --modules --incremental + +- - - + Update a single environment and specify a default branch override: - r10k deploy environment my_working_environment --puppetfile --default-branch-override default_branch_override + r10k deploy environment my_working_environment --modules --default-branch-override default_branch_override -This will update the given environment and update all contained modules, overrideing -the :default_branch entry in the Puppetfile of each module. This is used primarily to allow +This will update the given environment and update all contained modules, overriding +the :default_branch entry in the Puppetfile of each module. If the specified override branch is not +found, it will fall back to the normal default branch and attempt to use that. This is used primarily to allow automated r10k solutions using the control_branch pattern with a temporary branch deployment to -ensure the deployment is pushed to the correct module repository branch. Note that the :default_branch -is only ever utilized if the desired ref cannot be located. +ensure the deployment is pushed to the correct module repository branch. Note that the :default_branch and its +override are only ever used if the specific desired ref cannot be located. ### Deploying modules diff -Nru r10k-3.7.0/doc/dynamic-environments/workflow-guide.mkd r10k-4.0.0/doc/dynamic-environments/workflow-guide.mkd --- r10k-3.7.0/doc/dynamic-environments/workflow-guide.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/doc/dynamic-environments/workflow-guide.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -38,7 +38,7 @@ # Your modules: mod "custom_facts", - :git => "git://github.com/user/custom_facts" + :git => "https://github.com/user/custom_facts" ``` For any existing modules that you branched, add a reference to the new branch @@ -46,7 +46,7 @@ ``` mod "other_module", - :git => "git://github.com/user/other_module", + :git => "https://github.com/user/other_module", :ref => "feature" ``` @@ -159,7 +159,7 @@ ``` mod "other_module", - :git => "git://github.com/user/other_module", + :git => "https://github.com/user/other_module", :ref => "feature" ``` diff -Nru r10k-3.7.0/doc/faq.mkd r10k-4.0.0/doc/faq.mkd --- r10k-3.7.0/doc/faq.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/doc/faq.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -17,7 +17,7 @@ ``` $ cat /usr/local/bin/generate-puppet-types.sh -!#/bin/bash +#!/bin/bash for environment in $1; do /opt/puppetlabs/bin/puppet generate types --environment $environment diff -Nru r10k-3.7.0/doc/git/providers.mkd r10k-4.0.0/doc/git/providers.mkd --- r10k-3.7.0/doc/git/providers.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/doc/git/providers.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -56,6 +56,28 @@ private_key: '/root/.ssh/private_repo_id' ``` +### HTTPS Configuration + +Public HTTPS based Git repositories can be accessed with no additional settings. +For repos that do require authentication, the 'oauth_token' option may be provided. + +```yaml +git: + oauth_token: '/etc/puppetlabs/r10k/token' +``` + +If you have per repository access tokens you can add them with the repositories list. + +```yaml +git: + # default access token + oauth_token: '/etc/puppetlabs/r10k/token' + repositories: + - remote: "https://github.com/my_org/private_repo.git" + # access token for this repo only + oauth_token: '/etc/puppetlabs/r10k/private_repo_token' +``` + #### Supported transports with Rugged Rugged compiles libgit2 and and the Ruby bindings when the gem is installed. You diff -Nru r10k-3.7.0/doc/puppetfile.mkd r10k-4.0.0/doc/puppetfile.mkd --- r10k-3.7.0/doc/puppetfile.mkd 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/doc/puppetfile.mkd 2023-08-01 15:06:51.000000000 +0000 @@ -51,10 +51,9 @@ ### forge The `forge` setting specifies which server that Forge based modules are fetched -from. This is currently a noop and is provided for compatibility with -librarian-puppet. - -R10k supports setting the Forge to use _globally_ in `r10k.yaml`. see [Configuration](/doc/dynamic-environments/configuration.mkd#baseurl) for details. +from. This declaration is only respected if [`forge.allow_puppetfile_override`](/dynamic-environments/configuration.mkd#allow_puppetfile_override) +is set to true in the main `r10k.yaml`. Otherwise, use [`forge.baseurl`](/doc/dynamic-environments/configuration.mkd#baseurl) +to globally configure where modules should be downloaded from. ### moduledir @@ -109,11 +108,19 @@ mod 'puppetlabs/apache', :latest +An explicit type and/or version can be specified using the standard interface, +`:type` and `:version`. The `:source` parameter is not supported for individual +forge modules and will be ignored. + + mod 'puppetlabs/apache', + type: 'forge', + version: '6.0.0' + ### Git Git repositories that contain a Puppet module can be cloned and used as modules. When Git is used, the module version can be specified by using `:ref`, `:tag`, -`:commit`, and `:branch`. +`:commit`, `:branch`, or the standard interface parameter `:version`. When a module is installed using `:ref`, r10k uses some simple heuristics to determine the type of Git object that should be checked out. This can be used @@ -122,11 +129,18 @@ When a module is installed using `:tag` or `:commit`, r10k assumes that the given object is a tag or commit and can do some optimizations around fetching the object. If the tag or commit is already available r10k will skip network -operations when updating the repo, which can speed up install times. +operations when updating the repo, which can speed up install times. When +`:ref` is set to track `HEAD`, it will synchronize the module on each run. Module versions can also be specified using `:branch` to track a specific branch reference. +In r10k 3.x the default branch was hardcoded to `master`; in 4.x that was +removed. A `default_ref` can be specified in the r10k config to +to mimic that old behavior, but it is recommended to set the ref on a +per-module basis in the Puppetfile. Read [here](dynamic-environments/configuration.mkd#default_ref) for more info +on the `default_ref` setting. + #### Examples ```ruby @@ -153,6 +167,13 @@ mod 'apache', :git => 'https://github.com/puppetlabs/puppetlabs-apache', :branch => 'docs_experiment' + +# Install puppetlabs/apache and use standard interface parameters pinned to the +# '2098a17' commit. +mod 'puppetlabs-apache', + type: 'git', + source: 'https://github.com/puppetlabs/puppetlabs-apache', + version: '2098a17' ``` #### Control Repo Branch Tracking @@ -195,8 +216,8 @@ mod 'apache', :svn => 'https://github.com/puppetlabs/puppetlabs-apache/trunk' -If an SVN revision number is specified with `:rev` (or `:revision`), that -SVN revision will be kept checked out. +If an SVN revision number is specified with `:rev`, `:revision`, or `:version`, +that SVN revision will be kept checked out. mod 'apache', :svn => 'https://github.com/puppetlabs/puppetlabs-apache/trunk', @@ -206,6 +227,11 @@ :svn => 'https://github.com/puppetlabs/puppetlabs-apache/trunk', :revision => '154' + mod 'apache', + type: 'svn', + source: 'https://github.com/puppetlabs/puppetlabs-apache/trunk', + version: '154' + If the SVN repository requires credentials, you can supply the `:username` and `:password` options. @@ -219,6 +245,19 @@ choose to supply SVN credentials make sure that the system running r10k is appropriately secured. +### Tarball + +Modules can be installed from tarball archives. A tarball module must specify a source URL to retreive the tarball content from. A tarball module may optionally specify a sha256 checksum as the module version. + + mod 'puppetlabs-apache', + type: 'tarball', + source: 'https://repo.example.com/puppet/modules/puppetlabs-apache-7.0.0.tar.gz', + version: 'aedd6dc1a5136c6a1a1ec2f285df2a70b0fe4c9effb254b5a1f58116e4c1659e' # sha256 digest + +If no version is specified, a tarball will be downloaded from the given source and cached. The cache will not be invalidated until the source URL is changed, or a sha256 checksum version is provided. + +Tarball module content will be unpacked directly into an appropriately named module directory. For example, the puppetlabs-apache-7.0.0.tar.gz archive in the example above will be unpacked into `/modules/apache/`. + ### Local In the event you want to store locally written modules in your r10k-managed @@ -292,6 +331,19 @@ For more information see the [FAQ entry](faq.mkd#how-do-i-prevent-r10k-from-removing-modules-in-the-modules-directory-of-my-git-repository) on managing internal and external modules in the same directory. +### Per-Item spec dir deployment + +During deployment, r10k's default behavior is to delete the spec directory. The +Puppetfile can modify this per module, overriding settings from the default +r10k config. The following example sets the module to deploy the spec +directory. + +``` +mod 'apache', + :git => 'git@github.com:puppetlabs/puppetlabs-apache.git', + :exclude_spec => false +``` + ### Per-Item Install Path Git and SVN content types support installing into an alternate path without changing diff -Nru r10k-3.7.0/docker/.gitignore r10k-4.0.0/docker/.gitignore --- r10k-3.7.0/docker/.gitignore 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -TEST-rspec.xml diff -Nru r10k-3.7.0/docker/.rspec r10k-4.0.0/docker/.rspec --- r10k-3.7.0/docker/.rspec 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/.rspec 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ ---require pupperware/spec_helper ---format RspecJunitFormatter ---out TEST-rspec.xml ---format documentation diff -Nru r10k-3.7.0/docker/Gemfile r10k-4.0.0/docker/Gemfile --- r10k-3.7.0/docker/Gemfile 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/Gemfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -source "https://rubygems.org" - -gem 'pupperware', - :git => 'https://github.com/puppetlabs/pupperware.git', - :branch => 'master', - :glob => 'gem/*.gemspec' - -group :test do - gem 'rspec' - gem 'rspec_junit_formatter' -end diff -Nru r10k-3.7.0/docker/Makefile r10k-4.0.0/docker/Makefile --- r10k-3.7.0/docker/Makefile 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,91 +0,0 @@ -PUPPERWARE_ANALYTICS_STREAM ?= dev -NAMESPACE ?= puppet -git_describe = $(shell git describe --tags) -vcs_ref := $(shell git rev-parse HEAD) -build_date := $(shell date -u +%FT%T) -hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?) -hadolint_command := hadolint -hadolint_container := hadolint/hadolint:latest -alpine_version := 3.9 -export BUNDLE_PATH = $(PWD)/.bundle/gems -export BUNDLE_BIN = $(PWD)/.bundle/bin -export GEMFILE = $(PWD)/Gemfile -export DOCKER_BUILDKIT = 1 - -ifeq ($(IS_RELEASE),true) - VERSION ?= $(shell echo $(git_describe) | sed 's/-.*//') - PUBLISHED_VERSION ?= $(shell curl --silent 'https://rubygems.org/api/v1/gems/r10k.json' | jq '."version"' | tr -d '"') - CONTAINER_EXISTS = $(shell DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect $(NAMESPACE)/r10k:$(VERSION) > /dev/null 2>&1; echo $$?) -ifeq ($(CONTAINER_EXISTS),0) - SKIP_BUILD ?= true -else ifneq ($(VERSION),$(PUBLISHED_VERSION)) - SKIP_BUILD ?= true -endif - - LATEST_VERSION ?= latest - dockerfile := release.Dockerfile - dockerfile_context := r10k -else - VERSION ?= edge - IS_LATEST := false - dockerfile := Dockerfile - dockerfile_context := $(PWD)/.. -endif - -prep: - @git fetch --unshallow 2> /dev/null ||: - @git fetch origin 'refs/tags/*:refs/tags/*' -ifeq ($(SKIP_BUILD),true) - @echo "SKIP_BUILD is true, exiting with 1" - @exit 1 -endif - -lint: -ifeq ($(hadolint_available),0) - @$(hadolint_command) r10k/$(dockerfile) -else - @docker pull $(hadolint_container) - @docker run --rm -v $(PWD)/r10k/$(dockerfile):/Dockerfile -i $(hadolint_container) $(hadolint_command) Dockerfile -endif - -build: prep - docker pull alpine:$(alpine_version) - docker build \ - ${DOCKER_BUILD_FLAGS} \ - --build-arg alpine_version=$(alpine_version) \ - --build-arg vcs_ref=$(vcs_ref) \ - --build-arg build_date=$(build_date) \ - --build-arg version=$(VERSION) \ - --build-arg pupperware_analytics_stream=$(PUPPERWARE_ANALYTICS_STREAM) \ - --file r10k/$(dockerfile) \ - --tag $(NAMESPACE)/r10k:$(VERSION) $(dockerfile_context) -ifeq ($(IS_LATEST),true) - @docker tag $(NAMESPACE)/r10k:$(VERSION) puppet/r10k:$(LATEST_VERSION) -endif - -test: prep - @bundle install --path $$BUNDLE_PATH --gemfile $$GEMFILE --with test - @bundle update - @PUPPET_TEST_DOCKER_IMAGE=$(NAMESPACE)/r10k:$(VERSION) \ - bundle exec --gemfile $$GEMFILE \ - rspec spec - -push-image: prep - @docker push $(NAMESPACE)/r10k:$(VERSION) -ifeq ($(IS_LATEST),true) - @docker push $(NAMESPACE)/r10k:$(LATEST_VERSION) -endif - -push-readme: - @docker pull sheogorath/readme-to-dockerhub - @docker run --rm \ - -v $(PWD)/README.md:/data/README.md \ - -e DOCKERHUB_USERNAME="$(DOCKERHUB_USERNAME)" \ - -e DOCKERHUB_PASSWORD="$(DOCKERHUB_PASSWORD)" \ - -e DOCKERHUB_REPO_PREFIX=puppet \ - -e DOCKERHUB_REPO_NAME=r10k \ - sheogorath/readme-to-dockerhub - -publish: push-image push-readme - -.PHONY: lint build test prep publish push-image push-readme diff -Nru r10k-3.7.0/docker/README.md r10k-4.0.0/docker/README.md --- r10k-3.7.0/docker/README.md 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/README.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,28 +0,0 @@ -# [puppetlabs/r10k](https://github.com/puppetlabs/r10k) - -r10k on a Docker image. Based on Alpine 3.8. - -## Configuration - -The following environment variables are supported: - -- `PUPPERWARE_ANALYTICS_ENABLED` - - Set to 'true' to enable Google Analytics metrics. Defaults to 'false'. - -## Analytics Data Collection - -The r10k container collects usage data. This is disabled by default. You can enable it by passing `--env PUPPERWARE_ANALYTICS_ENABLED=true` -to your `docker run` command. - -### What data is collected? -* Version of the r10k container. -* Anonymized IP address is used by Google Analytics for Geolocation data, but the IP address is not collected. - -### Why does the r10k container collect data? - -We collect data to help us understand how the containers are used and make decisions about upcoming changes. - -### How can I opt out of r10k container data collection? - -This is disabled by default. diff -Nru r10k-3.7.0/docker/r10k/Dockerfile r10k-4.0.0/docker/r10k/Dockerfile --- r10k-3.7.0/docker/r10k/Dockerfile 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/r10k/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -ARG alpine_version=3.9 -FROM alpine:${alpine_version} as build - -# hadolint ignore=DL3018 -RUN apk add --no-cache ruby git && \ - mkdir /workspace -WORKDIR /workspace -COPY . /workspace -RUN gem build r10k.gemspec && \ - mv r10k*.gem r10k.gem - -FROM alpine:${alpine_version} - -ARG vcs_ref -ARG build_date -ARG version="3.1.0" -# Used by entrypoint to submit metrics to Google Analytics. -# Published images should use "production" for this build_arg. -ARG pupperware_analytics_stream="dev" -# required to schedule runs of "r10k" in K8s -ARG supercronic_version="0.1.9" -ARG supercronic_sha1sum="5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85" -ARG supercronic="supercronic-linux-amd64" -ARG supercronic_url="https://github.com/aptible/supercronic/releases/download/v$supercronic_version/$supercronic" - -LABEL org.label-schema.maintainer="Puppet Release Team " \ - org.label-schema.vendor="Puppet" \ - org.label-schema.url="https://github.com/puppetlabs/r10k" \ - org.label-schema.name="r10k" \ - org.label-schema.license="Apache-2.0" \ - org.label-schema.vcs-url="https://github.com/puppetlabs/r10k" \ - org.label-schema.schema-version="1.0" \ - org.label-schema.dockerfile="/Dockerfile" - -COPY docker/r10k/adduser.sh docker/r10k/docker-entrypoint.sh / -COPY docker/r10k/docker-entrypoint.d /docker-entrypoint.d - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["help"] - -# dynamic LABELs and ENV vars placed lower for the sake of Docker layer caching -ENV PUPPERWARE_ANALYTICS_STREAM="$pupperware_analytics_stream" - -LABEL org.label-schema.version="$version" \ - org.label-schema.vcs-ref="$vcs_ref" \ - org.label-schema.build-date="$build_date" - -COPY --from=build /workspace/r10k.gem / -SHELL ["/bin/ash", "-eo", "pipefail", "-c"] -# ignore apk and gem pinning -# hadolint ignore=DL3018,DL3028 -RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \ -# Add a puppet user to run r10k as for consistency with puppetserver - /adduser.sh && \ - chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \ - apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-json ruby-etc && \ - gem install --no-doc /r10k.gem && \ - rm -f /r10k.gem && \ - curl --fail --silent --show-error --location --remote-name "$supercronic_url" && \ - echo "${supercronic_sha1sum} ${supercronic}" | sha1sum -c - && \ - chmod +x "$supercronic" && \ - mv "$supercronic" "/usr/local/bin/${supercronic}" && \ - ln -s "/usr/local/bin/${supercronic}" /usr/local/bin/supercronic - -USER puppet -WORKDIR /home/puppet - -COPY docker/r10k/Dockerfile / diff -Nru r10k-3.7.0/docker/r10k/adduser.sh r10k-4.0.0/docker/r10k/adduser.sh --- r10k-3.7.0/docker/r10k/adduser.sh 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/r10k/adduser.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,13 +0,0 @@ -#!/bin/sh - -getent_string="$(getent group | grep -e ':999$')" -exit_code=$? - -if [ "$exit_code" = '0' ]; then - group="$(echo $getent_string | cut -d ':' -f1)" -else - addgroup -g 999 puppet - group='puppet' -fi - -adduser -G $group -D -u 999 puppet diff -Nru r10k-3.7.0/docker/r10k/docker-entrypoint.d/10-analytics.sh r10k-4.0.0/docker/r10k/docker-entrypoint.d/10-analytics.sh --- r10k-3.7.0/docker/r10k/docker-entrypoint.d/10-analytics.sh 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/r10k/docker-entrypoint.d/10-analytics.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,30 +0,0 @@ -#!/bin/sh - -if [ "${PUPPERWARE_ANALYTICS_ENABLED}" != "true" ]; then - # Don't print out any messages here since this is a CLI container - exit 0 -fi - -# See: https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters -# Tracking ID -tid=UA-132486246-5 -# Application Name -an=r10k -# Application Version -av=$(r10k version | cut -d ' ' -f 2) -# Anonymous Client ID -_file=/var/tmp/pwclientid -cid=$(cat $_file 2>/dev/null || (cat /proc/sys/kernel/random/uuid | tee $_file)) -# Event Category -ec=${PUPPERWARE_ANALYTICS_STREAM:-dev} -# Event Action -ea=start -# Anonymize ip -aip=1 - -_params="v=1&t=event&tid=${tid}&an=${an}&av=${av}&cid=${cid}&ec=${ec}&ea=${ea}&aip=${aip}" -_url="http://www.google-analytics.com/collect?${_params}" - -# Don't print out any messages here since this is a CLI container -curl --fail --silent --show-error --output /dev/null \ - -X POST -H "Content-Length: 0" $_url diff -Nru r10k-3.7.0/docker/r10k/docker-entrypoint.sh r10k-4.0.0/docker/r10k/docker-entrypoint.sh --- r10k-3.7.0/docker/r10k/docker-entrypoint.sh 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/r10k/docker-entrypoint.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -#! /bin/sh - -set -e - -for f in /docker-entrypoint.d/*.sh; do - # Don't print out any messages here since this is a CLI container - chmod +x "$f" - "$f" -done - -exec /usr/bin/r10k "$@" diff -Nru r10k-3.7.0/docker/r10k/release.Dockerfile r10k-4.0.0/docker/r10k/release.Dockerfile --- r10k-3.7.0/docker/r10k/release.Dockerfile 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/r10k/release.Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,55 +0,0 @@ -ARG alpine_version=3.9 -FROM alpine:${alpine_version} - -ARG vcs_ref -ARG build_date -ARG version="3.1.0" -# Used by entrypoint to submit metrics to Google Analytics. -# Published images should use "production" for this build_arg. -ARG pupperware_analytics_stream="dev" -# required to schedule runs of "r10k" in K8s -ARG supercronic_version="0.1.9" -ARG supercronic_sha1sum="5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85" -ARG supercronic="supercronic-linux-amd64" -ARG supercronic_url="https://github.com/aptible/supercronic/releases/download/v$supercronic_version/$supercronic" - -LABEL org.label-schema.maintainer="Puppet Release Team " \ - org.label-schema.vendor="Puppet" \ - org.label-schema.url="https://github.com/puppetlabs/r10k" \ - org.label-schema.name="r10k" \ - org.label-schema.license="Apache-2.0" \ - org.label-schema.vcs-url="https://github.com/puppetlabs/r10k" \ - org.label-schema.schema-version="1.0" \ - org.label-schema.dockerfile="/release.Dockerfile" - -COPY adduser.sh docker-entrypoint.sh / -COPY docker-entrypoint.d /docker-entrypoint.d - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["help"] - -# dyanmic LABELs and ENV vars placed lower for the sake of Docker layer caching -ENV PUPPERWARE_ANALYTICS_STREAM="$pupperware_analytics_stream" - -LABEL org.label-schema.version="$version" \ - org.label-schema.vcs-ref="$vcs_ref" \ - org.label-schema.build-date="$build_date" - -SHELL ["/bin/ash", "-eo", "pipefail", "-c"] -# ignore apk and gem pinning -# hadolint ignore=DL3018,DL3028 -RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \ - /adduser.sh && \ - chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \ - apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-dev make gcc musl-dev && \ - gem install --no-doc r10k:"$version" json etc && \ - curl --fail --silent --show-error --location --remote-name "$supercronic_url" && \ - echo "${supercronic_sha1sum} ${supercronic}" | sha1sum -c - && \ - chmod +x "$supercronic" && \ - mv "$supercronic" "/usr/local/bin/${supercronic}" && \ - ln -s "/usr/local/bin/${supercronic}" /usr/local/bin/supercronic - -USER puppet -WORKDIR /home/puppet - -COPY release.Dockerfile / diff -Nru r10k-3.7.0/docker/spec/dockerfile_spec.rb r10k-4.0.0/docker/spec/dockerfile_spec.rb --- r10k-3.7.0/docker/spec/dockerfile_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/spec/dockerfile_spec.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,43 +0,0 @@ -require 'rspec/core' -require 'fileutils' -require 'open3' - -SPEC_DIRECTORY = File.dirname(__FILE__) - -describe 'r10k container' do - include Pupperware::SpecHelpers - def run_r10k(command) - run_command("docker run --detach \ - --volume #{File.join(SPEC_DIRECTORY, 'fixtures')}:/home/puppet/test \ - #{@image} #{command} \ - --verbose \ - --trace \ - --puppetfile test/Puppetfile") - end - - before(:all) do - @image = require_test_image - end - - after(:all) do - FileUtils.rm_rf(File.join(SPEC_DIRECTORY, 'fixtures', 'modules')) - end - - it 'should validate the Puppetfile' do - result = run_r10k('puppetfile check') - container = result[:stdout].chomp - wait_on_container_exit(container) - expect(get_container_exit_code(container)).to eq(0) - emit_log(container) - teardown_container(container) - end - - it 'should install the Puppetfile' do - result = run_r10k('puppetfile install') - container = result[:stdout].chomp - wait_on_container_exit(container) - expect(get_container_exit_code(container)).to eq(0) - emit_log(container) - teardown_container(container) - end -end diff -Nru r10k-3.7.0/docker/spec/fixtures/Puppetfile r10k-4.0.0/docker/spec/fixtures/Puppetfile --- r10k-3.7.0/docker/spec/fixtures/Puppetfile 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/docker/spec/fixtures/Puppetfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -moduledir '/tmp/modules' -mod 'puppetlabs/ntp' diff -Nru r10k-3.7.0/integration/Rakefile r10k-4.0.0/integration/Rakefile --- r10k-3.7.0/integration/Rakefile 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/Rakefile 2023-08-01 15:06:51.000000000 +0000 @@ -1,4 +1,5 @@ require 'rototiller' +require 'fileutils' namespace :ci do namespace :test do @@ -59,13 +60,14 @@ desc 'Generate a host configuration used by Beaker' rototiller_task :beaker_hostgenerator do |t| if ENV['BEAKER_HOST'].nil? + FileUtils.mkdir_p 'configs' t.add_command do |c| c.name = 'beaker-hostgenerator' c.argument = '> configs/generated' end # This is a hack :( - t.add_flag(:name => '', :default => 'centos6-64mdca-64.fa', :override_env => 'TEST_TARGET') + t.add_flag(:name => '', :default => 'centos7-64mdca-64.fa', :override_env => 'TEST_TARGET') t.add_flag(:name => '--global-config', :default => '{forge_host=forge-aio01-petest.puppetlabs.com}', :override_env => 'BHG_GLOBAL_CONFIG') end diff -Nru r10k-3.7.0/integration/files/pre-suite/git_config.pp.erb r10k-4.0.0/integration/files/pre-suite/git_config.pp.erb --- r10k-3.7.0/integration/files/pre-suite/git_config.pp.erb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/files/pre-suite/git_config.pp.erb 2023-08-01 15:06:51.000000000 +0000 @@ -1,4 +1,4 @@ -$git_package = $osfamily ? { +$git_package = $facts['os']['family'] ? { 'Debian' => 'git-core', default => 'git' } diff -Nru r10k-3.7.0/integration/pre-suite/10_git_config.rb r10k-4.0.0/integration/pre-suite/10_git_config.rb --- r10k-3.7.0/integration/pre-suite/10_git_config.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/pre-suite/10_git_config.rb 2023-08-01 15:06:51.000000000 +0000 @@ -23,9 +23,6 @@ fail_test('This pre-suite requires PE 3.7 or above!') if pe_version < 3.7 #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Read module path' on(master, puppet('config print basemodulepath')) do |result| (result.stdout.include? ':') ? separator = ':' : separator = ';' diff -Nru r10k-3.7.0/integration/tests/Puppetfile/HTTP_PROXY_affects_git_source.rb r10k-4.0.0/integration/tests/Puppetfile/HTTP_PROXY_affects_git_source.rb --- r10k-3.7.0/integration/tests/Puppetfile/HTTP_PROXY_affects_git_source.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/Puppetfile/HTTP_PROXY_affects_git_source.rb 2023-08-01 15:06:51.000000000 +0000 @@ -19,7 +19,7 @@ puppetfile =<<-EOS mod 'motd', - :git => 'https://github.com/puppetlabs/puppetlabs-motd' + :git => 'https://github.com/puppetlabs/puppetlabs-motd', :branch => 'main' EOS proxy_env_value = 'http://ferritsarebest.net:3219' @@ -36,7 +36,7 @@ CONF teardown do - master.clear_env_var('HTTP_PROXY') + master.clear_env_var('HTTPS_PROXY') step 'Restore Original "r10k" Config' on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}") @@ -45,7 +45,7 @@ clean_up_r10k(master, last_commit, git_environments_path) end -master.add_env_var('HTTP_PROXY', proxy_env_value) +master.add_env_var('HTTPS_PROXY', proxy_env_value) step 'Backup Current "r10k" Config' on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") @@ -64,7 +64,8 @@ #test on(master, "#{r10k_fqp} deploy environment -p", :accept_all_exit_codes => true) do |r| - regex = /(Couldn't|Could not) resolve proxy.*ferritsarebest\.net/i + # Rugged as of 0.28 has a different error message than shellgit + regex = /((failed to resolve address for)|(Could not resolve proxy:)) ferritsarebest\.net/ assert(r.exit_code == 1, 'expected error code was not observed') assert_match(regex, r.stderr, 'The expected error message was not observed' ) end diff -Nru r10k-3.7.0/integration/tests/basic_functionality/basic_deployment.rb r10k-4.0.0/integration/tests/basic_functionality/basic_deployment.rb --- r10k-3.7.0/integration/tests/basic_functionality/basic_deployment.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/integration/tests/basic_functionality/basic_deployment.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,176 @@ +require 'git_utils' +require 'r10k_utils' +require 'master_manipulator' + + +test_name 'Basic Environment Deployment Workflows' + +# This isn't a block because we want to use the local variables throughout the file +step 'init' + @env_path = on(master, puppet('config print environmentpath')).stdout.rstrip + r10k_fqp = get_r10k_fqp(master) + + control_repo_gitdir = '/git_repos/environments.git' + control_repo_worktree = '/root/environments' + last_commit = git_last_commit(master, control_repo_worktree) + git_provider = ENV['GIT_PROVIDER'] + + config_path = get_r10k_config_file_path(master) + config_backup_path = "#{config_path}.bak" + + puppetfile1 =<<-EOS + mod 'puppetlabs/apache', '0.10.0' + mod 'puppetlabs/stdlib', '8.0.0' + EOS + + r10k_conf = <<-CONF + cachedir: '/var/cache/r10k' + git: + provider: '#{git_provider}' + sources: + control: + basedir: "#{@env_path}" + remote: "#{control_repo_gitdir}" + deploy: + purge_levels: ['deployment','environment','puppetfile'] + + CONF + + +def and_stdlib_is_correct + metadata_path = "#{@env_path}/production/modules/stdlib/metadata.json" + on(master, "test -f #{metadata_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'stdlib content has been inappropriately purged') + end + metadata_info = JSON.parse(on(master, "cat #{metadata_path}").stdout) + assert(metadata_info['version'] == '8.0.0', 'stdlib deployed to wrong version') +end + +teardown do + on(master, "mv #{config_backup_path} #{config_path}") + clean_up_r10k(master, last_commit, control_repo_worktree) +end + +step 'Set up r10k and control repo' do + + # Backup and replace r10k config + on(master, "mv #{config_path} #{config_backup_path}") + create_remote_file(master, config_path, r10k_conf) + + # Place our Puppetfile in the control repo's production branch + git_on(master, 'checkout production', control_repo_worktree) + create_remote_file(master, "#{control_repo_worktree}/Puppetfile", puppetfile1) + git_add_commit_push(master, 'production', 'add Puppetfile for Basic Deployment test', control_repo_worktree) + + # Ensure the production environment will be deployed anew + on(master, "rm -rf #{@env_path}/production") +end + +test_path = "#{@env_path}/production/modules/apache/metadata.json" +step 'Test initial environment deploy works' do + on(master, "#{r10k_fqp} deploy environment production --verbose=info") do |result| + assert(result.output =~ /.*Deploying module to .*apache.*/, 'Did not log apache deployment') + assert(result.output =~ /.*Deploying module to .*stdlib.*/, 'Did not log stdlib deployment') + end + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module in Puppetfile was not installed') + end + + and_stdlib_is_correct +end + +original_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + +step 'Test second run of deploy updates control repo, but leaves moduledir untouched' do + puppetfile2 =<<-EOS + # Current latest of apache is 6.5.1 as of writing this test + mod 'puppetlabs/apache', :latest + mod 'puppetlabs/stdlib', '8.0.0' + mod 'puppetlabs/concat', '7.0.0' + EOS + + git_on(master, 'checkout production', control_repo_worktree) + create_remote_file(master, "#{control_repo_worktree}/Puppetfile", puppetfile2) + git_add_commit_push(master, 'production', 'add Puppetfile for Basic Deployment test', control_repo_worktree) + + on(master, "#{r10k_fqp} deploy environment production --verbose=info") do |result| + refute(result.output =~ /.*Deploying module to .*apache.*/, 'Inappropriately updated apache') + refute(result.output =~ /.*Deploying module to .*stdlib.*/, 'Inappropriately updated stdlib') + end + + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module content in Puppetfile was inappropriately purged') + end + + new_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + on(master, "cat #{@env_path}/production/Puppetfile | grep ':latest'", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Puppetfile not updated on subsequent r10k deploys') + end + + assert(original_apache_info['version'] == new_apache_info['version'] && + new_apache_info['version'] == '0.10.0', + 'Module content updated on subsequent r10k invocations w/o providing --modules') + + on(master, "test -f #{@env_path}/production/modules/concat/metadata.json", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 1, 'Module content deployed on subsequent r10k invocation w/o providing --modules') + end + + and_stdlib_is_correct +end + +step 'Test --modules updates modules' do + on(master, "#{r10k_fqp} deploy environment production --modules --verbose=info") do |result| + assert(result.output =~ /.*Deploying module to .*apache.*/, 'Did not log apache deployment') + assert(result.output =~ /.*Deploying module to .*stdlib.*/, 'Did not log stdlib deployment') + assert(result.output =~ /.*Deploying module to .*concat.*/, 'Did not log concat deployment') + end + + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module content in Puppetfile was inappropriately purged') + end + + on(master, "test -f #{@env_path}/production/modules/concat/metadata.json", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'New module content was not deployed when providing --modules') + end + + new_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + apache_major_version = new_apache_info['version'].split('.').first.to_i + assert(apache_major_version > 5, 'Module not updated correctly using --modules') + + and_stdlib_is_correct +end + +step 'Test --modules --incremental deploys changed & dynamic modules, but not unchanged, static modules' do + puppetfile3 =<<-EOS + # Current latest of apache is 6.5.1 as of writing this test + mod 'puppetlabs/apache', :latest + mod 'puppetlabs/stdlib', '8.0.0' + mod 'puppetlabs/concat', '7.1.0' + EOS + + git_on(master, 'checkout production', control_repo_worktree) + create_remote_file(master, "#{control_repo_worktree}/Puppetfile", puppetfile3) + git_add_commit_push(master, 'production', 'add Puppetfile for Basic Deployment test', control_repo_worktree) + + on(master, "#{r10k_fqp} deploy environment production --modules --incremental --verbose=debug1") do |result| + assert(result.output =~ /.*Deploying module to .*apache.*/, 'Did not log apache deployment') + assert(result.output =~ /.*Deploying module to .*concat.*/, 'Did not log concat deployment') + assert(result.output =~ /.*Not updating module stdlib, assuming content unchanged.*/, 'Did not log notice of skipping stdlib') + end + + on(master, "test -f #{test_path}", accept_all_exit_codes: true) do |result| + assert(result.exit_code == 0, 'Expected module content in Puppetfile was inappropriately purged') + end + + new_apache_info = JSON.parse(on(master, "cat #{test_path}").stdout) + apache_major_version = new_apache_info['version'].split('.').first.to_i + assert(apache_major_version > 5, 'Module not updated correctly using --modules & --incremental') + + concat_info = JSON.parse(on(master, "cat #{@env_path}/production/modules/concat/metadata.json").stdout) + concat_minor_version = concat_info['version'].split('.')[1].to_i + assert(concat_minor_version == 1, 'Module not updated correctly using --modules & --incremental') + + and_stdlib_is_correct +end + + diff -Nru r10k-3.7.0/integration/tests/basic_functionality/install_pe_only_module_with_puppetfile.rb r10k-4.0.0/integration/tests/basic_functionality/install_pe_only_module_with_puppetfile.rb --- r10k-3.7.0/integration/tests/basic_functionality/install_pe_only_module_with_puppetfile.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/basic_functionality/install_pe_only_module_with_puppetfile.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,83 +0,0 @@ -require 'git_utils' -require 'r10k_utils' -require 'master_manipulator' -test_name 'RK-158 - C92362 - Install a PE-only module from forge' - -#Init -env_path = on(master, puppet('config print environmentpath')).stdout.rstrip -r10k_fqp = get_r10k_fqp(master) -master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip - -git_repo_path = '/git_repos' -git_repo_name = 'environments' -git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git") -git_environments_path = '/root/environments' -last_commit = git_last_commit(master, git_environments_path) -git_provider = ENV['GIT_PROVIDER'] || 'shellgit' - -r10k_config_path = get_r10k_config_file_path(master) -r10k_config_bak_path = "#{r10k_config_path}.bak" - -#In-line files -r10k_conf = <<-CONF -cachedir: '/var/cache/r10k' -git: - provider: '#{git_provider}' -sources: - control: - basedir: "#{env_path}" - remote: "#{git_control_remote}" -CONF - -#Manifest -site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp') -site_pp = create_site_pp(master_certname, ' include peonly') - -# Verification -notify_message_regex = /I am in the production environment, this is a PE only module/ - -#Teardown -teardown do - step 'remove license file' - on(master, 'rm -f /etc/puppetlabs/license.key') - - step 'Restore Original "r10k" Config' - on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}") - - step 'cleanup r10k' - clean_up_r10k(master, last_commit, git_environments_path) -end - -#Setup -step 'Stub the forge' -stub_forge_on(master) - -step 'Backup a Valid "r10k" Config' -on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") - -step 'Update the "r10k" Config' -create_remote_file(master, r10k_config_path, r10k_conf) - -step 'Download license file from artifactory' -curl_on(master, 'https://artifactory.delivery.puppetlabs.net/artifactory/generic/r10k_test_license.key -o /etc/puppetlabs/license.key') - -step 'Inject New "site.pp" to the "production" Environment' -inject_site_pp(master, site_pp_path, site_pp) - -step 'Copy Puppetfile to "production" Environment Git Repo' -create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "ztr-peonly"') - -step 'Push Changes' -git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path) - -#Tests -step 'Deploy "production" Environment via r10k' -on(master, "#{r10k_fqp} deploy environment -p") - -agents.each do |agent| - step "Run Puppet Agent" - on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 2) do |result| - assert_no_match(/Error:/, result.stderr, 'Unexpected error was detected!') - assert_match(notify_message_regex, result.stdout, 'Expected message not found!') - end -end diff -Nru r10k-3.7.0/integration/tests/basic_functionality/proxy_specified_in_configuration.rb r10k-4.0.0/integration/tests/basic_functionality/proxy_specified_in_configuration.rb --- r10k-3.7.0/integration/tests/basic_functionality/proxy_specified_in_configuration.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/basic_functionality/proxy_specified_in_configuration.rb 2023-08-01 15:06:51.000000000 +0000 @@ -50,7 +50,7 @@ CONF #Verification -squid_log_regex = /CONNECT forgeapi.puppetlabs.com:443/ +squid_log_regex = /CONNECT forgeapi.puppet(labs)?.com:443/ #Teardown teardown do diff -Nru r10k-3.7.0/integration/tests/basic_functionality/proxy_with_pe_only_module.rb r10k-4.0.0/integration/tests/basic_functionality/proxy_with_pe_only_module.rb --- r10k-3.7.0/integration/tests/basic_functionality/proxy_with_pe_only_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/basic_functionality/proxy_with_pe_only_module.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,128 +0,0 @@ -require 'git_utils' -require 'r10k_utils' -require 'master_manipulator' -test_name 'RK-242 '#'- C87652 - Specify the proxy in the r10k.yaml' - -confine(:to, :platform => ['el', 'sles']) - -#Init -master_platform = fact_on(master, 'osfamily') -master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip -env_path = on(master, puppet('config print environmentpath')).stdout.rstrip -r10k_fqp = get_r10k_fqp(master) - -git_repo_path = '/git_repos' -git_repo_name = 'environments' -git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git") -git_environments_path = '/root/environments' -last_commit = git_last_commit(master, git_environments_path) -git_provider = ENV['GIT_PROVIDER'] - -local_files_root_path = ENV['FILES'] || 'files' - -git_manifest_template_path = File.join(local_files_root_path, 'pre-suite', 'git_config.pp.erb') -git_manifest = ERB.new(File.read(git_manifest_template_path)).result(binding) - -r10k_config_path = get_r10k_config_file_path(master) -r10k_config_bak_path = "#{r10k_config_path}.bak" - -case master_platform - when 'RedHat' - pkg_manager = 'yum' - when 'Suse' - pkg_manager = 'zypper' -end - -install_squid = "#{pkg_manager} install -y squid" -remove_squid = "#{pkg_manager} remove -y squid" -squid_log = "/var/log/squid/access.log" - -#In-line files -r10k_conf = <<-CONF -cachedir: '/var/cache/r10k' -git: - provider: '#{git_provider}' -sources: - control: - basedir: "#{env_path}" - remote: "#{git_control_remote}" -forge: - proxy: "http://#{master.hostname}:3128" -CONF - -#Manifest -site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp') -site_pp = create_site_pp(master_certname, ' include peonly') - -#Verification -squid_log_regex = /CONNECT forgeapi.puppetlabs.com:443/ -notify_message_regex = /I am in the production environment, this is a PE only module/ - -#Teardown -teardown do - step 'remove license file' - on(master, 'rm -f /etc/puppetlabs/license.key') - - step 'Restore "git" Package' - on(master, puppet('apply'), :stdin => git_manifest, :acceptable_exit_codes => [0,2]) - - step 'Restore Original "r10k" Config' - on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}") - - clean_up_r10k(master, last_commit, git_environments_path) - - step 'Remove Squid' - on(master, puppet("apply -e 'service {'squid' : ensure => stopped}'")) - on(master, remove_squid) -end - -#Setup -step 'Stub the forge' -stub_forge_on(master) - -step 'Backup Current "r10k" Config' -on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") - -step 'Update the "r10k" Config' -create_remote_file(master, r10k_config_path, r10k_conf) - -step 'Download license file from artifactory' -curl_on(master, 'https://artifactory.delivery.puppetlabs.net/artifactory/generic/r10k_test_license.key -o /etc/puppetlabs/license.key') - -step 'Checkout "production" Branch' -git_on(master, 'checkout production', git_environments_path) - -step 'Inject New "site.pp" to the "production" Environment' -inject_site_pp(master, site_pp_path, site_pp) - -step 'Copy Puppetfile to "production" Environment with PE only module' -create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "ztr-peonly"') - -step 'Push Changes' -git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path) - -step 'Install and configure squid proxy' -on(master, install_squid) - -step 'turn off the firewall' -on(master, puppet("apply -e 'service {'iptables' : ensure => stopped}'")) - -step 'start squid proxy' -on(master, puppet("apply -e 'service {'squid' : ensure => running}'")) - -#Tests -step 'Deploy "production" Environment via r10k' -on(master, "#{r10k_fqp} deploy environment -p") - -step 'Read the squid logs' -on(master, "cat #{squid_log}") do |result| - assert_match(squid_log_regex, result.stdout, 'Proxy logs did not indicate use of the proxy.') -end - -agents.each do |agent| - step "Run Puppet Agent" - on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => [0,2]) do |result| - assert_no_match(/Error:/, result.stderr, 'Unexpected error was detected!') - assert_match(notify_message_regex, result.stdout, 'Expected message not found!') - end -end diff -Nru r10k-3.7.0/integration/tests/basic_functionality/proxy_with_puppetfile.rb r10k-4.0.0/integration/tests/basic_functionality/proxy_with_puppetfile.rb --- r10k-3.7.0/integration/tests/basic_functionality/proxy_with_puppetfile.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/basic_functionality/proxy_with_puppetfile.rb 2023-08-01 15:06:51.000000000 +0000 @@ -21,7 +21,7 @@ squid_log = "/var/log/squid/access.log" #Verification -squid_log_regex = /CONNECT forgeapi.puppetlabs.com:443/ +squid_log_regex = /CONNECT forgeapi.puppet(labs)?.com:443/ #Teardown teardown do diff -Nru r10k-3.7.0/integration/tests/command_line/deploy_env_without_mod_update.rb r10k-4.0.0/integration/tests/command_line/deploy_env_without_mod_update.rb --- r10k-3.7.0/integration/tests/command_line/deploy_env_without_mod_update.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/command_line/deploy_env_without_mod_update.rb 2023-08-01 15:06:51.000000000 +0000 @@ -46,9 +46,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Inject New "site.pp" to the "production" Environment' inject_site_pp(master, site_pp_path, site_pp) diff -Nru r10k-3.7.0/integration/tests/command_line/negative/neg_deploy_env_with_module_update.rb r10k-4.0.0/integration/tests/command_line/negative/neg_deploy_env_with_module_update.rb --- r10k-3.7.0/integration/tests/command_line/negative/neg_deploy_env_with_module_update.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/command_line/negative/neg_deploy_env_with_module_update.rb 2023-08-01 15:06:51.000000000 +0000 @@ -47,9 +47,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Inject New "site.pp" to the "production" Environment' inject_site_pp(master, site_pp_path, site_pp) diff -Nru r10k-3.7.0/integration/tests/git_source/HTTP_proxy_and_git_source.rb r10k-4.0.0/integration/tests/git_source/HTTP_proxy_and_git_source.rb --- r10k-3.7.0/integration/tests/git_source/HTTP_proxy_and_git_source.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/git_source/HTTP_proxy_and_git_source.rb 2023-08-01 15:06:51.000000000 +0000 @@ -16,28 +16,24 @@ puppetfile =<<-EOS mod 'motd', - :git => 'git://github.com/puppetlabs/puppetlabs-motd' + :git => 'https://github.com/puppetlabs/puppetlabs-motd', :branch => 'main' EOS -proxy_env_value = 'http://cattastic.net:3219' - #In-line files r10k_conf = <<-CONF cachedir: '/var/cache/r10k' git: provider: '#{git_provider}' repositories: - - remote: 'http://example.com/fake_git_source.git' + - remote: 'https://something.else/repo' proxy: 'http://foooooooo.unresolvable:3128' sources: control: basedir: "#{env_path}" - remote: "http://example.com/fake_git_source.git" + remote: 'https://something.else/repo' CONF teardown do - master.clear_env_var('HTTP_PROXY') - step 'Restore Original "r10k" Config' on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}") @@ -45,8 +41,6 @@ clean_up_r10k(master, last_commit, git_environments_path) end -master.add_env_var('HTTP_PROXY', proxy_env_value) - step 'Backup Current "r10k" Config' on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") @@ -64,7 +58,8 @@ #test on(master, "#{r10k_fqp} deploy environment -p", :accept_all_exit_codes => true) do |r| - regex = /proxy.*foooooooo\.unresolvable/ + # Rugged as of 0.28 has a different error message than shellgit + regex = /((failed to resolve address for)|(Could not resolve proxy:)) foooooooo\.unresolvable/ assert(r.exit_code == 1, 'expected error code was not observed') assert_match(regex, r.stderr, 'The expected error message was not observed' ) end diff -Nru r10k-3.7.0/integration/tests/git_source/git_source_git.rb r10k-4.0.0/integration/tests/git_source/git_source_git.rb --- r10k-3.7.0/integration/tests/git_source/git_source_git.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/git_source/git_source_git.rb 2023-08-01 15:06:51.000000000 +0000 @@ -84,9 +84,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Backup Current "r10k" Config' on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") diff -Nru r10k-3.7.0/integration/tests/git_source/git_source_repeated_remote.rb r10k-4.0.0/integration/tests/git_source/git_source_repeated_remote.rb --- r10k-3.7.0/integration/tests/git_source/git_source_repeated_remote.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/git_source/git_source_repeated_remote.rb 2023-08-01 15:06:51.000000000 +0000 @@ -29,12 +29,12 @@ # Install the same module in two different places puppetfile = <<-EOS mod 'prod_apache', - :git => 'git://github.com/puppetlabs/puppetlabs-apache.git', - :branch => 'master' + :git => 'https://github.com/puppetlabs/puppetlabs-apache.git', + :tag => 'v6.0.0' mod 'test_apache', - :git => 'git://github.com/puppetlabs/puppetlabs-apache.git', - :branch => 'master' + :git => 'https://github.com/puppetlabs/puppetlabs-apache.git', + :tag => 'v6.0.0' EOS teardown do @@ -44,9 +44,6 @@ clean_up_r10k(master, last_commit, git_environments_path) end -step 'Stub the forge' -stub_forge_on(master) - step 'Backup Current "r10k" Config' on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") diff -Nru r10k-3.7.0/integration/tests/git_source/git_source_submodule.rb r10k-4.0.0/integration/tests/git_source/git_source_submodule.rb --- r10k-3.7.0/integration/tests/git_source/git_source_submodule.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/git_source/git_source_submodule.rb 2023-08-01 15:06:51.000000000 +0000 @@ -44,7 +44,7 @@ git_add_commit_push(master, 'master', 'Add module.', git_clone_module_path) step 'Add "helloworld" Module Git Repo as Submodule' -on(master, "cd #{git_environments_path};git submodule add file://#{git_repo_module_path} dist") +on(master, "cd #{git_environments_path};git -c protocol.file.allow=always submodule add file://#{git_repo_module_path} dist") step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/git_source/negative/neg_git_unauthorized_ssh.rb r10k-4.0.0/integration/tests/git_source/negative/neg_git_unauthorized_ssh.rb --- r10k-3.7.0/integration/tests/git_source/negative/neg_git_unauthorized_ssh.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/git_source/negative/neg_git_unauthorized_ssh.rb 2023-08-01 15:06:51.000000000 +0000 @@ -68,6 +68,6 @@ #Tests step 'Attempt to Deploy via r10k' -on(master, "#{r10k_fqp} deploy environment -v", :acceptable_exit_codes => 1) do |result| +on(master, "SSH_AUTH_SOCK= SSH_CONNECTION= SSH_CLIENT= #{r10k_fqp} deploy environment -v", :acceptable_exit_codes => 1) do |result| assert_match(error_message_regex, result.stderr, 'Expected message not found!') end diff -Nru r10k-3.7.0/integration/tests/purging/content_not_purged_at_root.rb r10k-4.0.0/integration/tests/purging/content_not_purged_at_root.rb --- r10k-3.7.0/integration/tests/purging/content_not_purged_at_root.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/purging/content_not_purged_at_root.rb 2023-08-01 15:06:51.000000000 +0000 @@ -30,12 +30,12 @@ puppetfile = <<-EOS mod 'non_module_object_1', :install_path => './', - :git => 'git://github.com/puppetlabs/control-repo.git', + :git => 'https://github.com/puppetlabs/control-repo.git', :branch => 'production' mod 'non_module_object_2', :install_path => '', - :git => 'git://github.com/puppetlabs/control-repo.git', + :git => 'https://github.com/puppetlabs/control-repo.git', :branch => 'production' EOS @@ -50,9 +50,6 @@ clean_up_r10k(master, last_commit, git_environments_path) end -step 'Stub the forge' -stub_forge_on(master) - step 'Backup Current "r10k" Config' on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") diff -Nru r10k-3.7.0/integration/tests/purging/default_purging.rb r10k-4.0.0/integration/tests/purging/default_purging.rb --- r10k-3.7.0/integration/tests/purging/default_purging.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/purging/default_purging.rb 2023-08-01 15:06:51.000000000 +0000 @@ -51,9 +51,6 @@ fake_file_c_to_be_purged = "#{fake_environment_path_c}/fakefile3.txt" # initalize file content -step 'Stub the forge' -stub_forge_on(master) - step 'Backup Current "r10k" Config' on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") diff -Nru r10k-3.7.0/integration/tests/purging/does_not_purge_files_on_allowlist.rb r10k-4.0.0/integration/tests/purging/does_not_purge_files_on_allowlist.rb --- r10k-3.7.0/integration/tests/purging/does_not_purge_files_on_allowlist.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/integration/tests/purging/does_not_purge_files_on_allowlist.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,90 @@ +require 'git_utils' +require 'r10k_utils' +require 'master_manipulator' +test_name 'RK-257 - C98046 - r10k does not purge files on allowlist' + +#Init +env_path = on(master, puppet('config print environmentpath')).stdout.rstrip +r10k_fqp = get_r10k_fqp(master) +git_environments_path = '/root/environments' + +git_repo_path = '/git_repos' +git_repo_name = 'environments' +git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git") + +last_commit = git_last_commit(master, git_environments_path) +git_provider = ENV['GIT_PROVIDER'] + +r10k_config_path = get_r10k_config_file_path(master) +r10k_config_bak_path = "#{r10k_config_path}.bak" + +teardown do + step 'Restore Original "r10k" Config' + on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}") + + clean_up_r10k(master, last_commit, git_environments_path) +end + +# initalize file content +step 'Backup Current "r10k" Config' +on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") + +r10k_conf = <<-CONF +cachedir: '/var/cache/r10k' +git: + provider: '#{git_provider}' +sources: + control: + basedir: "#{env_path}" + remote: "#{git_control_remote}" +deploy: + purge_levels: ['deployment', 'environment', 'puppetfile'] + purge_allowlist: ['**/*.pp'] +CONF + +step 'Update the "r10k" Config' +create_remote_file(master, r10k_config_path, r10k_conf) + +step 'Copy Puppetfile to "production" Environment Git Repo' +create_remote_file(master, "#{git_environments_path}/Puppetfile", "mod 'puppetlabs-stdlib' \n mod 'puppetlabs-motd'") + +step 'Push Changes' +git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path) + +step 'Deploy production' +on(master, "#{r10k_fqp} deploy environment -p") + +step 'commit a new Puppetfile to production' +create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "puppetlabs-motd"') + +step 'Push Changes' +git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path) + +step 'create test pp files' +do_not_purge = [ + "/etc/puppetlabs/code/environments/production/environment_level.pp", + "/etc/puppetlabs/code/environments/production/site/environment_level.pp" +].each do |file| + create_remote_file(master, file, 'this is a test') +end + +purge = [ + "/etc/puppetlabs/code/environments/production/environment_level.zz", + "/etc/puppetlabs/code/environments/production/site/environment_level.zz" +].each do |file| + create_remote_file(master, file, 'this is a test') +end + +#TEST +step 'Deploy again and check files' +on(master, "#{r10k_fqp} deploy environment -p") + +purge.each do |file| + assert_message = "The file #{file}\n was not purged, it was expected to be" + assert(on(master, "test -f #{file}", :accept_all_exit_codes => true).exit_code == 1, assert_message) +end + +do_not_purge.each do |file| + assert_message = "The file #{file}\n was purged, it was not expected to be" + assert(on(master, "test -f #{file}", :accept_all_exit_codes => true).exit_code == 0, assert_message) +end diff -Nru r10k-3.7.0/integration/tests/purging/does_not_purge_files_on_white_list.rb r10k-4.0.0/integration/tests/purging/does_not_purge_files_on_white_list.rb --- r10k-3.7.0/integration/tests/purging/does_not_purge_files_on_white_list.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/purging/does_not_purge_files_on_white_list.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,93 +0,0 @@ -require 'git_utils' -require 'r10k_utils' -require 'master_manipulator' -test_name 'RK-257 - C98046 - r10k does not purge files on whitelist' - -#Init -env_path = on(master, puppet('config print environmentpath')).stdout.rstrip -r10k_fqp = get_r10k_fqp(master) -git_environments_path = '/root/environments' - -git_repo_path = '/git_repos' -git_repo_name = 'environments' -git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git") - -last_commit = git_last_commit(master, git_environments_path) -git_provider = ENV['GIT_PROVIDER'] - -r10k_config_path = get_r10k_config_file_path(master) -r10k_config_bak_path = "#{r10k_config_path}.bak" - -teardown do - step 'Restore Original "r10k" Config' - on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}") - - clean_up_r10k(master, last_commit, git_environments_path) -end - -# initalize file content -step 'Stub the forge' -stub_forge_on(master) - -step 'Backup Current "r10k" Config' -on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") - -r10k_conf = <<-CONF -cachedir: '/var/cache/r10k' -git: - provider: '#{git_provider}' -sources: - control: - basedir: "#{env_path}" - remote: "#{git_control_remote}" -deploy: - purge_levels: ['deployment', 'environment', 'puppetfile'] - purge_whitelist: ['**/*.pp'] -CONF - -step 'Update the "r10k" Config' -create_remote_file(master, r10k_config_path, r10k_conf) - -step 'Copy Puppetfile to "production" Environment Git Repo' -create_remote_file(master, "#{git_environments_path}/Puppetfile", "mod 'puppetlabs-stdlib' \n mod 'puppetlabs-motd'") - -step 'Push Changes' -git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path) - -step 'Deploy production' -on(master, "#{r10k_fqp} deploy environment -p") - -step 'commit a new Puppetfile to production' -create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "puppetlabs-motd"') - -step 'Push Changes' -git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path) - -step 'create test pp files' -do_not_purge = [ - "/etc/puppetlabs/code/environments/production/environment_level.pp", - "/etc/puppetlabs/code/environments/production/site/environment_level.pp" -].each do |file| - create_remote_file(master, file, 'this is a test') -end - -purge = [ - "/etc/puppetlabs/code/environments/production/environment_level.zz", - "/etc/puppetlabs/code/environments/production/site/environment_level.zz" -].each do |file| - create_remote_file(master, file, 'this is a test') -end - -#TEST -step 'Deploy again and check files' -on(master, "#{r10k_fqp} deploy environment -p") - -purge.each do |file| - assert_message = "The file #{file}\n was not purged, it was expected to be" - assert(on(master, "test -f #{file}", :accept_all_exit_codes => true).exit_code == 1, assert_message) -end - -do_not_purge.each do |file| - assert_message = "The file #{file}\n was purged, it was not expected to be" - assert(on(master, "test -f #{file}", :accept_all_exit_codes => true).exit_code == 0, assert_message) -end diff -Nru r10k-3.7.0/integration/tests/purging/invalid_whitelist_types.rb r10k-4.0.0/integration/tests/purging/invalid_whitelist_types.rb --- r10k-3.7.0/integration/tests/purging/invalid_whitelist_types.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/purging/invalid_whitelist_types.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,63 +0,0 @@ -require 'git_utils' -require 'r10k_utils' -require 'master_manipulator' -test_name 'RK-257 - C98043 - verify default whitelist only accepts strings or array of strings' - -#Init -env_path = on(master, puppet('config print environmentpath')).stdout.rstrip -r10k_fqp = get_r10k_fqp(master) -git_environments_path = '/root/environments' - -git_repo_path = '/git_repos' -git_repo_name = 'environments' -git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git") - -last_commit = git_last_commit(master, git_environments_path) -git_provider = ENV['GIT_PROVIDER'] - -r10k_config_path = get_r10k_config_file_path(master) -r10k_config_bak_path = "#{r10k_config_path}.bak" - -#invalid content to test -hash_whitelist = '{:cats => \'cats.txt\'}' -invalid_array_content_whitelist = '[\'cats.txt\', [:broken]]' - -teardown do - step 'Restore Original "r10k" Config' - on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}") - - clean_up_r10k(master, last_commit, git_environments_path) -end - -# initalize file content -step 'Stub the forge' -stub_forge_on(master) - -step 'Backup Current "r10k" Config' -on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") - -[hash_whitelist, invalid_array_content_whitelist].each do |whitelist_content| - r10k_conf = <<-CONF -cachedir: '/var/cache/r10k' -git: - provider: '#{git_provider}' -sources: - control: - basedir: "#{env_path}" - remote: "#{git_control_remote}" -deploy: - purge_whitelist: #{whitelist_content} - CONF - - step 'Update the "r10k" Config' - create_remote_file(master, r10k_config_path, r10k_conf) - - step 'Deploy r10k, and verify that invalid whitelist content causes error' - on(master, "#{r10k_fqp} deploy environment -p", :accept_all_exit_codes => true) do |result| - error = /did not find expected node content while parsing a flow node/ - error_message = 'whitelist content did not generate expected error' - expect_failure('RK-263') do - assert_no_match(result.stdout, error, error_message) - end - end -end diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -27,7 +27,8 @@ puppet_file = <<-PUPPETFILE mod "puppetlabs/motd" mod 'puppetlabs/stdlib', - :git => 'git://github.com/puppetlabs/puppetlabs-stdlib.git' + :git => 'https://github.com/puppetlabs/puppetlabs-stdlib.git', + :tag => 'v7.0.1' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') @@ -61,9 +62,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - env_names.each do |env| if env == 'production' step "Checkout \"#{env}\" Branch" diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb 2023-08-01 15:06:51.000000000 +0000 @@ -31,7 +31,8 @@ moduledir '#{@module_path}' mod "puppetlabs/motd" mod 'puppetlabs/stdlib', - :git => 'git://github.com/puppetlabs/puppetlabs-stdlib.git' + :git => 'https://github.com/puppetlabs/puppetlabs-stdlib.git', + :tag => 'v7.0.1' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') @@ -65,9 +66,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - env_names.each do |env| if env == 'production' step "Checkout \"#{env}\" Branch" diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -65,7 +65,7 @@ '/git_repos_alt/environments_alt.git', '/root/environments_alt', '/root/environments_alt/Puppetfile', - 'mod "puppetlabs/stdlib", :git => "git://github.com/puppetlabs/puppetlabs-stdlib.git"', + 'mod "puppetlabs/stdlib", :git => "https://github.com/puppetlabs/puppetlabs-stdlib.git", :tag => "v7.0.1"', '/root/environments_alt/manifests/site.pp', create_site_pp(master_certname, stage_env_manifest) ), @@ -102,9 +102,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Backup Current "r10k" Config' on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}") diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_forge_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_forge_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_forge_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_forge_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -29,9 +29,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -15,7 +15,7 @@ #File puppet_file = <<-PUPPETFILE -mod 'broken', :git => 'git://github.com/puppetlabs/puppetlabs-broken' +mod 'broken', :git => 'https://github.com/puppetlabs/puppetlabs-broken' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_bad_git_module_ref.rb 2023-08-01 15:06:51.000000000 +0000 @@ -12,7 +12,7 @@ #File puppet_file = <<-PUPPETFILE mod 'broken', - :git => 'git://github.com/puppetlabs/puppetlabs-motd', + :git => 'https://github.com/puppetlabs/puppetlabs-motd', :ref => 'does_not_exist' PUPPETFILE diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_duplicate_module_names.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_duplicate_module_names.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_duplicate_module_names.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_duplicate_module_names.rb 2023-08-01 15:06:51.000000000 +0000 @@ -25,9 +25,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_inaccessible_forge.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_inaccessible_forge.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_inaccessible_forge.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_inaccessible_forge.rb 2023-08-01 15:06:51.000000000 +0000 @@ -19,7 +19,7 @@ puppet_file_path = File.join(git_environments_path, 'Puppetfile') #Verification -error_message_regex = /Error: Could not connect via HTTPS to https:\/\/forgeapi.puppetlabs.com/ +error_message_regex = /Error: Could not connect via HTTPS to https:\/\/forgeapi.puppet(labs)?.com/ #Teardown teardown do @@ -34,7 +34,8 @@ on(master, "mv #{hosts_file_path} #{hosts_file_path}.bak") step 'Point Forge Hostname to Localhost' -on(master, "echo '127.0.0.1 forgeapi.puppetlabs.com' > #{hosts_file_path}") +on(master, "echo '127.0.0.1 forgeapi.puppet.com' > #{hosts_file_path}") +on(master, "echo '127.0.0.1 forgeapi.puppetlabs.com' >> #{hosts_file_path}") step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_module_specified_at_deleted_release.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_module_specified_at_deleted_release.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_module_specified_at_deleted_release.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_module_specified_at_deleted_release.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,49 +0,0 @@ -require 'git_utils' -require 'r10k_utils' -test_name 'CODEMGMT-127 - C64121 - Attempt to Deploy Environment with Forge Module Specified at Deleted Release' - -#This test uses the spotty module at https://forge-aio01-petest.puppetlabs.com//puppetlabs/spotty, with valid released 0.1.0 and 0.3.0 versions, and deleted 0.2.0 and 0.4.0 versions. - -#Init -git_environments_path = '/root/environments' -last_commit = git_last_commit(master, git_environments_path) -r10k_fqp = get_r10k_fqp(master) - -#Verification -if get_puppet_version(master) < 4.0 - error_notification_regex = /No releases matching '0.2.0'/ -else - error_notification_regex = /error.* -> The module release puppetlabs-spotty-0.2.0 does not exist on/i -end - -#File -puppet_file = <<-PUPPETFILE -mod "puppetlabs/spotty", '0.2.0' -PUPPETFILE - -puppet_file_path = File.join(git_environments_path, 'Puppetfile') - -#Teardown -teardown do - clean_up_r10k(master, last_commit, git_environments_path) -end - -#Setup -step 'Stub Forge on Master' -stub_forge_on(master) - -#Tests -step 'Checkout "production" Branch' -git_on(master, 'checkout production', git_environments_path) - -step 'Create "Puppetfile" for the "production" Environment' -create_remote_file(master, puppet_file_path, puppet_file) - -step 'Push Changes' -git_add_commit_push(master, 'production', 'Add module.', git_environments_path) - -#Tests -step "Deploy production environment via r10k with module specified at deleted version" -on(master, "#{r10k_fqp} deploy environment -p -v", :acceptable_exit_codes => 1) do |result| - assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!') -end diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -10,7 +10,7 @@ r10k_fqp = get_r10k_fqp(master) #Verification -error_notification_regex = /Does 'puppetlabs-regret' have at least one published release?/ +error_notification_regex = /(The module puppetlabs-regret does not appear to have any published releases)|(module puppetlabs-regret does not exist on)/ #File puppet_file = <<-PUPPETFILE @@ -24,10 +24,6 @@ clean_up_r10k(master, last_commit, git_environments_path) end -#Setup -step 'Stub Forge on Master' -stub_forge_on(master) - #Tests step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) @@ -40,12 +36,6 @@ #Tests step "Deploy production environment via r10k with specified module deleted" -on(master, "#{r10k_fqp} deploy environment -p -v", :acceptable_exit_codes => 1) do |result| - if get_puppet_version(master) < 4.0 - assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!') - else - expect_failure('expected to fail due to RK-135') do - assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!') - end - end +on(master, "#{r10k_fqp} deploy environment -p -v --trace", :acceptable_exit_codes => 1) do |result| + assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!') end diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -28,7 +28,8 @@ puppet_file = <<-PUPPETFILE mod "puppetlabs/motd" mod 'puppetlabs/inifile', - :git => 'git://github.com/puppetlabs/puppetlabs-inifile' + :git => 'https://github.com/puppetlabs/puppetlabs-inifile', + :tag => 'v5.0.1' PUPPETFILE puppet_file_path = File.join(git_environments_path, 'Puppetfile') @@ -63,9 +64,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -45,9 +45,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_module_already_installed.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_module_already_installed.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_module_already_installed.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_module_already_installed.rb 2023-08-01 15:06:51.000000000 +0000 @@ -44,9 +44,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Add motd module from the forge using the PMT' on(master, puppet('module', 'install', 'puppetlabs-motd', '--modulepath', forge_module_path)) diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_module_last_release_deleted.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_module_last_release_deleted.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_module_last_release_deleted.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_module_last_release_deleted.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -require 'git_utils' -require 'r10k_utils' -test_name 'CODEMGMT-127 - C64120 - Single Environment with Forge Module where Latest Release has been Deleted' - -#This test uses the spotty module at https://forge-aio01-petest.puppetlabs.com//puppetlabs/spotty, which has valid 0.1.0 and 0.3.0 versions, and deleted 0.2.0 and 0.4.0 versions. - -#Init -env_path = on(master, puppet('config print environmentpath')).stdout.rstrip -prod_env_modules_path = File.join(env_path, 'production', 'modules') -r10k_fqp = get_r10k_fqp(master) - -git_environments_path = '/root/environments' -last_commit = git_last_commit(master, git_environments_path) - -#Verification -module_contents = 'Version 3' -module_contents_regex = /\A#{module_contents}\n\z/ - -module_contents_path = File.join(prod_env_modules_path, 'spotty', 'manifests', 'init.pp') -module_version_filepath = File.join(prod_env_modules_path, 'spotty', 'metadata.json') -module_version_3_regex = /"0.3.0"/ - -#File -puppet_file = <<-PUPPETFILE -mod "puppetlabs/spotty" -PUPPETFILE - -puppet_file_path = File.join(git_environments_path, 'Puppetfile') - -#Teardown -teardown do - clean_up_r10k(master, last_commit, git_environments_path) -end - -#Setup -step 'Stub Forge on Master' -stub_forge_on(master) - -#Tests -step 'Checkout "production" Branch' -git_on(master, 'checkout production', git_environments_path) - -step 'Create "Puppetfile" for the "production" Environment' -create_remote_file(master, puppet_file_path, puppet_file) - -step 'Push Changes' -git_add_commit_push(master, 'production', 'Add module.', git_environments_path) - -#Tests -step 'Deploy "production" Environment via r10k with modules' -on(master, "#{r10k_fqp} deploy environment -p -v") - -agents.each do |agent| - step "Run Puppet Agent" - on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => [0,2]) do |result| - assert_no_match(/Error:/, result.stderr, 'Unexpected error was detected!') - end - - step 'Verify Contents' - on(master, "cat #{module_contents_path}") do |result| - assert_match(module_contents, result.stdout, 'File Content is Invalid') - end - - step 'Verify Version' - on(master, "grep version #{module_version_filepath}") do |result| - assert_match(module_version_3_regex, result.stdout, 'File Content is Invalid') - end -end diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,12 +1,15 @@ require 'git_utils' require 'r10k_utils' require 'master_manipulator' -test_name 'CODEMGMT-73 - C63184 - Single Environment Purge Unmanaged Modules' +test_name 'CODEMGMT-78 - Puppetfile Purge --puppetfile & --moduledir flag usage' #Init master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip -git_environments_path = '/root/environments' -last_commit = git_last_commit(master, git_environments_path) +environments_path = on(master, puppet('config', 'print', 'environmentpath')).stdout.strip +moduledir = File.join(environments_path, 'production', 'modules') +puppetfile_path = File.join(environments_path, 'production', 'Puppetfile') +git_remote_environments_path = '/root/environments' +last_commit = git_last_commit(master, git_remote_environments_path) r10k_fqp = get_r10k_fqp(master) #Verification @@ -14,14 +17,12 @@ motd_contents = 'Hello!' motd_contents_regex = /\A#{motd_contents}\z/ -error_message_regex = /Blah/ - #File -puppet_file = <<-PUPPETFILE +puppetfile = <<-PUPPETFILE mod "puppetlabs/xinetd" PUPPETFILE -puppet_file_path = File.join(git_environments_path, 'Puppetfile') +remote_puppetfile_path = File.join(git_remote_environments_path, 'Puppetfile') #Manifest manifest = <<-MANIFEST @@ -30,43 +31,37 @@ } MANIFEST -site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp') +remote_site_pp_path = File.join(git_remote_environments_path, 'manifests', 'site.pp') site_pp = create_site_pp(master_certname, manifest) #Teardown teardown do - clean_up_r10k(master, last_commit, git_environments_path) + clean_up_r10k(master, last_commit, git_remote_environments_path) step 'Remove "/etc/motd" File' on(agents, "rm -rf #{motd_path}") end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Checkout "production" Branch' -git_on(master, 'checkout production', git_environments_path) - -step 'Manually Install the "motd" Module from the Forge' -on(master, puppet('module install puppetlabs-motd')) +git_on(master, 'checkout production', git_remote_environments_path) step 'Create "Puppetfile" for the "production" Environment' -create_remote_file(master, puppet_file_path, puppet_file) +create_remote_file(master, remote_puppetfile_path, puppetfile) step 'Inject New "site.pp" to the "production" Environment' -inject_site_pp(master, site_pp_path, site_pp) +inject_site_pp(master, remote_site_pp_path, site_pp) step 'Push Changes' -git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_environments_path) +git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_remote_environments_path) -#Tests step 'Deploy Environments via r10k' -on(master, "#{r10k_fqp} deploy environment -v -p") +on(master, "#{r10k_fqp} deploy environment --modules --verbose debug --trace") -step 'Plug-in Sync Agents' -on(agents, puppet("plugin download --server #{master}")) +step 'Manually Install the "motd" Module from the Forge' +on(master, puppet("module install puppetlabs-motd --modulepath #{moduledir}")) +#Tests agents.each do |agent| step 'Run Puppet Agent Against "production" Environment' on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 2) do |result| @@ -80,14 +75,12 @@ end step 'Use r10k to Purge Unmanaged Modules' -on(master, "#{r10k_fqp} puppetfile purge -v", :acceptable_exit_codes => 1) +on(master, "#{r10k_fqp} puppetfile purge --puppetfile #{puppetfile_path} --moduledir #{moduledir} --verbose debug --trace") #Agent will fail because r10k will purge the "motd" module agents.each do |agent| step 'Attempt to Run Puppet Agent' - on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 0) do |result| - expect_failure('Expected to fail due to CODEMGMT-78') do - assert_match(error_message_regex, result.stderr, 'Expected error was not detected!') - end + on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 1) do |result| + assert_match(/Could not find declared class motd/, result.stderr, 'Module was not purged') end end diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_switch_forge_git_module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -27,7 +27,7 @@ puppet_file_git = <<-PUPPETFILE mod "puppetlabs/motd", - :git => 'git://github.com/puppetlabs/puppetlabs-motd', + :git => 'https://github.com/puppetlabs/puppetlabs-motd', :tag => '1.2.0' PUPPETFILE @@ -57,9 +57,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_upgrade_forge_mod_revert_change.rb r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_upgrade_forge_mod_revert_change.rb --- r10k-3.7.0/integration/tests/user_scenario/basic_workflow/single_env_upgrade_forge_mod_revert_change.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/basic_workflow/single_env_upgrade_forge_mod_revert_change.rb 2023-08-01 15:06:51.000000000 +0000 @@ -60,9 +60,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - step 'Checkout "production" Branch' git_on(master, 'checkout production', git_environments_path) diff -Nru r10k-3.7.0/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb r10k-4.0.0/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb --- r10k-3.7.0/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb 2023-08-01 15:06:51.000000000 +0000 @@ -24,7 +24,7 @@ stage_env_notify_message_regex = /#{stage_env_notify_message}/ #Verification for "test" Environment -test_env_error_message_regex = /Error:.*Could not.*environment '?test'?/ +test_env_message_regex = /Environment 'test' not found on server/ #Verification for "temp" Environment test_env_notify_message_regex = /I am in the temp environment/ @@ -65,9 +65,6 @@ end #Setup -step 'Stub Forge on Master' -stub_forge_on(master) - initial_env_names.each do |env| if env == 'production' step "Checkout \"#{env}\" Branch" @@ -157,7 +154,7 @@ end step 'Attempt to Run Puppet Agent Against "test" Environment' - on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result| - assert_match(test_env_error_message_regex, result.stderr, 'Expected error was not detected!') + on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result| + assert_match(test_env_message_regex, result.stdout, 'Expected message not found!') end end diff -Nru r10k-3.7.0/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb r10k-4.0.0/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb --- r10k-3.7.0/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb 2023-08-01 15:06:51.000000000 +0000 @@ -15,7 +15,7 @@ #Verification notify_message_regex = /I am in the production environment/ -stage_env_error_message_regex = /Error:.*Could not.*environment '?stage'?/ +stage_env_message_regex = /Environment 'stage' not found on server/ #Manifest site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp') @@ -83,8 +83,8 @@ end step 'Attempt to Run Puppet Agent Against "stage" Environment' - on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 1) do |result| - assert_match(stage_env_error_message_regex, result.stderr, 'Expected error was not detected!') + on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 2) do |result| + assert_match(stage_env_message_regex, result.stdout, 'Expected message not found!') end end diff -Nru r10k-3.7.0/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb r10k-4.0.0/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb --- r10k-3.7.0/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb 2023-08-01 15:06:51.000000000 +0000 @@ -22,7 +22,7 @@ notify_message_prod_env_regex = /I am in the production environment/ notify_message_test_env_regex = /I am in the test environment/ removal_message_test_env_regex = /Removing unmanaged path.*test/ -error_message_regex = /Could not retrieve (catalog from remote server|information from environment test)/ +missing_message_regex = /Environment 'test' not found on server/ #Teardown teardown do @@ -72,7 +72,7 @@ agents.each do |agent| step 'Run Puppet Agent Against "test" Environment' - on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result| - assert_match(error_message_regex, result.stderr, 'Expected message not found!') + on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result| + assert_match(missing_message_regex, result.stdout, 'Expected message not found!') end end diff -Nru r10k-3.7.0/lib/r10k/action/base.rb r10k-4.0.0/lib/r10k/action/base.rb --- r10k-3.7.0/lib/r10k/action/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/base.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,5 +1,5 @@ -require 'r10k/util/setopts' require 'r10k/logging' +require 'r10k/util/setopts' module R10K module Action @@ -10,6 +10,16 @@ attr_accessor :settings + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version def initialize(opts, argv, settings = {}) @opts = opts @argv = argv diff -Nru r10k-3.7.0/lib/r10k/action/deploy/deploy_helpers.rb r10k-4.0.0/lib/r10k/action/deploy/deploy_helpers.rb --- r10k-3.7.0/lib/r10k/action/deploy/deploy_helpers.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/deploy/deploy_helpers.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,8 +1,12 @@ +require 'r10k/logging' + module R10K module Action module Deploy module DeployHelpers + include R10K::Logging + # Ensure that a config file has been found (and presumably loaded) and exit # with a helpful error if it hasn't. # diff -Nru r10k-3.7.0/lib/r10k/action/deploy/display.rb r10k-4.0.0/lib/r10k/action/deploy/display.rb --- r10k-3.7.0/lib/r10k/action/deploy/display.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/deploy/display.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,6 +1,6 @@ -require 'r10k/deployment' require 'r10k/action/base' require 'r10k/action/deploy/deploy_helpers' +require 'r10k/deployment' module R10K module Action @@ -9,17 +9,48 @@ include R10K::Action::Deploy::DeployHelpers + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version + def initialize(opts, argv, settings = {}) + super + + @settings = @settings.merge({ + overrides: { + environments: { + preload_environments: @fetch, + requested_environments: @argv.map { |arg| arg.gsub(/\W/, '_') } + }, + modules: {}, + output: { + format: @format, + trace: @trace, + detail: @detail + }, + purging: {} + } + }) + end + def call expect_config! deployment = R10K::Deployment.new(@settings) - if @fetch + if @settings.dig(:overrides, :environments, :preload_environments) deployment.preload! + deployment.validate! end - output = { :sources => deployment.sources.map { |source| source_info(source, @argv) } } + output = { :sources => deployment.sources.map { |source| source_info(source, @settings.dig(:overrides, :environments, :requested_environments)) } } - case @format + case @settings.dig(:overrides, :output, :format) when 'json' then json_format(output) else yaml_format(output) end @@ -27,7 +58,7 @@ # exit 0 true rescue => e - logger.error R10K::Errors::Formatting.format_exception(e, @trace) + logger.error R10K::Errors::Formatting.format_exception(e, @settings.dig(:overrides, :output, :trace)) false end @@ -43,7 +74,7 @@ puts output.to_yaml end - def source_info(source, argv=[]) + def source_info(source, requested_environments = []) source_info = { :name => source.name, :basedir => source.basedir, @@ -52,28 +83,30 @@ source_info[:prefix] = source.prefix if source.prefix source_info[:remote] = source.remote if source.respond_to?(:remote) - env_list = source.environments.select { |env| argv.empty? || argv.include?(env.name) } + select_all_envs = requested_environments.empty? + env_list = source.environments.select { |env| select_all_envs || requested_environments.include?(env.name) } source_info[:environments] = env_list.map { |env| environment_info(env) } source_info end def environment_info(env) - if !@puppetfile && !@detail + modules = @settings.dig(:overrides, :environments, :deploy_modules) + if !modules && !@settings.dig(:overrides, :output, :detail) env.dirname else env_info = env.info.merge({ :status => (env.status rescue nil), }) - env_info[:modules] = env.modules.map { |mod| module_info(mod) } if @puppetfile + env_info[:modules] = env.modules.map { |mod| module_info(mod) } if modules env_info end end def module_info(mod) - if @detail + if @settings.dig(:overrides, :output, :detail) { :name => mod.title, :properties => mod.properties } else mod.title @@ -81,7 +114,13 @@ end def allowed_initialize_opts - super.merge(puppetfile: :self, detail: :self, format: :self, fetch: :self) + super.merge({ + puppetfile: :modules, + modules: :self, + detail: :self, + format: :self, + fetch: :self + }) end end end diff -Nru r10k-3.7.0/lib/r10k/action/deploy/environment.rb r10k-4.0.0/lib/r10k/action/deploy/environment.rb --- r10k-3.7.0/lib/r10k/action/deploy/environment.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/deploy/environment.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,9 +1,9 @@ -require 'r10k/util/setopts' -require 'r10k/deployment' -require 'r10k/logging' -require 'r10k/action/visitor' require 'r10k/action/base' require 'r10k/action/deploy/deploy_helpers' +require 'r10k/action/visitor' +require 'r10k/deployment' +require 'r10k/util/setopts' + require 'json' module R10K @@ -12,35 +12,75 @@ class Environment < R10K::Action::Base include R10K::Action::Deploy::DeployHelpers + include R10K::Action::Visitor + # Deprecated attr_reader :force - def initialize(opts, argv, settings = nil) - settings ||= {} - @purge_levels = settings.fetch(:deploy, {}).fetch(:purge_levels, []) - @user_purge_whitelist = settings.fetch(:deploy, {}).fetch(:purge_whitelist, []) - @generate_types = settings.fetch(:deploy, {}).fetch(:generate_types, false) + attr_reader :settings + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version + def initialize(opts, argv, settings = {}) super - # @force here is used to make it easier to reason about - @force = !@no_force - @argv = @argv.map { |arg| arg.gsub(/\W/,'_') } + # instance variables below are set by the super class based on the + # spec of #allowed_initialize_opts and any command line flags. This + # gives a preference order of cli flags > config files > defaults. + @settings = @settings.merge({ + overrides: { + environments: { + requested_environments: @argv.map { |arg| arg.gsub(/\W/,'_') }, + default_branch_override: @default_branch_override, + generate_types: @generate_types || settings.dig(:deploy, :generate_types) || false, + preload_environments: true, + incremental: @incremental + }, + modules: { + default_ref: settings.dig(:git, :default_ref), + exclude_spec: settings.dig(:deploy, :exclude_spec), + requested_modules: [], + deploy_modules: @modules, + pool_size: @settings[:pool_size] || 4, + force: !@no_force, # force here is used to make it easier to reason about + }, + purging: { + purge_levels: settings.dig(:deploy, :purge_levels) || [], + purge_allowlist: settings.dig(:deploy, :purge_allowlist) || [] + }, + forge: { + allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false + }, + output: {} + } + }) end def call @visit_ok = true - expect_config! - deployment = R10K::Deployment.new(@settings) - check_write_lock!(@settings) + begin + expect_config! + deployment = R10K::Deployment.new(@settings) + check_write_lock!(@settings) + + deployment.accept(self) + rescue => e + @visit_ok = false + logger.error R10K::Errors::Formatting.format_exception(e, @trace) + end - deployment.accept(self) @visit_ok end - include R10K::Action::Visitor - private def visit_deployment(deployment) @@ -48,10 +88,12 @@ # sources then we can't fully enumerate all environments which # could be dangerous. If this fails then an exception will be raised # and execution will be halted. - deployment.preload! - deployment.validate! + if @settings.dig(:overrides, :environments, :preload_environments) + deployment.preload! + deployment.validate! + end - undeployable = undeployable_environment_names(deployment.environments, @argv) + undeployable = undeployable_environment_names(deployment.environments, @settings.dig(:overrides, :environments, :requested_environments)) if !undeployable.empty? @visit_ok = false logger.error _("Environment(s) \'%{environments}\' cannot be found in any source and will not be deployed.") % {environments: undeployable.join(", ")} @@ -59,17 +101,22 @@ yield - if @purge_levels.include?(:deployment) + if @settings.dig(:overrides, :purging, :purge_levels).include?(:deployment) logger.debug("Purging unmanaged environments for deployment...") + deployment.sources.each do |source| + source.reload! + end deployment.purge! end ensure if (postcmd = @settings[:postrun]) if postcmd.grep('$modifiedenvs').any? envs = deployment.environments.map { |e| e.dirname } - envs.reject! { |e| !@argv.include?(e) } if @argv.any? + requested_envs = @settings.dig(:overrides, :environments, :requested_environments) + envs.reject! { |e| !requested_envs.include?(e) } if requested_envs.any? postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs.join(' ')) } end + logger.debug _("Executing postrun command.") subproc = R10K::Util::Subprocess.new(postcmd) subproc.logger = logger subproc.execute @@ -81,7 +128,8 @@ end def visit_environment(environment) - if !(@argv.empty? || @argv.any? { |name| environment.dirname == name }) + requested_envs = @settings.dig(:overrides, :environments, :requested_environments) + if !(requested_envs.empty? || requested_envs.any? { |name| environment.dirname == name }) logger.debug1(_("Environment %{env_dir} does not match environment name filter, skipping") % {env_dir: environment.dirname}) return end @@ -95,28 +143,31 @@ environment.sync logger.info _("Environment %{env_dir} is now at %{env_signature}") % {env_dir: environment.dirname, env_signature: environment.signature} - if status == :absent || @puppetfile + if status == :absent || @settings.dig(:overrides, :modules, :deploy_modules) if status == :absent logger.debug(_("Environment %{env_dir} is new, updating all modules") % {env_dir: environment.dirname}) end previous_ok = @visit_ok @visit_ok = true - yield + + environment.deploy + @environment_ok = @visit_ok @visit_ok &&= previous_ok end - if @purge_levels.include?(:environment) + + if @settings.dig(:overrides, :purging, :purge_levels).include?(:environment) if @visit_ok logger.debug("Purging unmanaged content for environment '#{environment.dirname}'...") - environment.purge!(:recurse => true, :whitelist => environment.whitelist(@user_purge_whitelist)) + environment.purge!(:recurse => true, :whitelist => environment.whitelist(@settings.dig(:overrides, :purging, :purge_allowlist))) else logger.debug("Not purging unmanaged content for environment '#{environment.dirname}' due to prior deploy failures.") end end - if @generate_types + if @settings.dig(:overrides, :environments, :generate_types) if @environment_ok logger.debug("Generating puppet types for environment '#{environment.dirname}'...") environment.generate_types! @@ -128,34 +179,21 @@ write_environment_info!(environment, started_at, @visit_ok) end - def visit_puppetfile(puppetfile) - puppetfile.load(@opts[:'default-branch-override']) - - yield - - if @purge_levels.include?(:puppetfile) - logger.debug("Purging unmanaged Puppetfile content for environment '#{puppetfile.environment.dirname}'...") - puppetfile.purge! - end - end - - def visit_module(mod) - logger.info _("Deploying %{origin} content %{path}") % {origin: mod.origin, path: mod.path} - mod.sync(force: @force) - end - def write_environment_info!(environment, started_at, success) - module_deploys = [] - begin - environment.modules.each do |mod| - name = mod.name - version = mod.version - sha = mod.repo.head rescue nil - module_deploys.push({:name => name, :version => version, :sha => sha}) + module_deploys = + begin + environment.modules.map do |mod| + props = mod.properties + { + name: mod.name, + version: props[:expected], + sha: props[:type] == :git ? props[:actual] : nil + } + end + rescue + logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}") + [] end - rescue - logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}") - end # make this file write as atomic as possible in pure ruby final = "#{environment.path}/.r10k-deploy.json" @@ -183,13 +221,21 @@ end def allowed_initialize_opts - super.merge(puppetfile: :self, + super.merge(puppetfile: :modules, + modules: :self, cachedir: :self, + incremental: :self, 'no-force': :self, + 'exclude-spec': :self, 'generate-types': :self, 'puppet-path': :self, 'puppet-conf': :self, - 'default-branch-override': :self) + 'private-key': :self, + 'oauth-token': :self, + 'default-branch-override': :self, + 'github-app-id': :self, + 'github-app-key': :self, + 'github-app-ttl': :self) end end end diff -Nru r10k-3.7.0/lib/r10k/action/deploy/module.rb r10k-4.0.0/lib/r10k/action/deploy/module.rb --- r10k-3.7.0/lib/r10k/action/deploy/module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/deploy/module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,7 +1,7 @@ -require 'r10k/deployment' -require 'r10k/action/visitor' require 'r10k/action/base' require 'r10k/action/deploy/deploy_helpers' +require 'r10k/action/visitor' +require 'r10k/deployment' module R10K module Action @@ -9,35 +9,89 @@ class Module < R10K::Action::Base include R10K::Action::Deploy::DeployHelpers + include R10K::Action::Visitor + # Deprecated attr_reader :force - def initialize(opts, argv, settings = nil) - settings ||= {} + attr_reader :settings + # @param opts [Hash] A hash of options defined in #allowed_initialized_opts + # and managed by the SetOps mixin within the Action::Base class. + # Corresponds to the CLI flags and options. + # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like + # collection of the remaining arguments to the CLI invocation (after + # removing flags and options). + # @param settings [Hash] A hash of configuration loaded from the relevant + # config (r10k.yaml). + # + # @note All arguments will be required in the next major version + def initialize(opts, argv, settings = {}) super - # @force here is used to make it easier to reason about - @force = !@no_force + requested_env = @opts[:environment] ? [@opts[:environment].gsub(/\W/, '_')] : [] + @modified_envs = [] + + @settings = @settings.merge({ + overrides: { + environments: { + requested_environments: requested_env, + generate_types: @generate_types + }, + modules: { + default_ref: settings.dig(:git, :default_ref), + exclude_spec: settings.dig(:deploy, :exclude_spec), + pool_size: @settings[:pool_size] || 4, + requested_modules: @argv.map.to_a, + # force here is used to make it easier to reason about + force: !@no_force + }, + forge: { + allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false + }, + purging: {}, + output: {} + } + }) end def call @visit_ok = true + begin + expect_config! + deployment = R10K::Deployment.new(@settings) + check_write_lock!(@settings) + + deployment.accept(self) + rescue => e + @visit_ok = false + logger.error R10K::Errors::Formatting.format_exception(e, @trace) + end - expect_config! - deployment = R10K::Deployment.new(@settings) - check_write_lock!(@settings) - - deployment.accept(self) @visit_ok end - include R10K::Action::Visitor - private def visit_deployment(deployment) yield + ensure + if (postcmd = @settings[:postrun]) + if @modified_envs.any? + envs_to_run = @modified_envs.join(' ') + logger.debug _("Running postrun command for environments: %{envs_to_run}.") % { envs_to_run: envs_to_run } + + if postcmd.grep('$modifiedenvs').any? + postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs_to_run) } + end + + subproc = R10K::Util::Subprocess.new(postcmd) + subproc.logger = logger + subproc.execute + else + logger.debug _("No environments were modified, not executing postrun command.") + end + end end def visit_source(source) @@ -45,39 +99,40 @@ end def visit_environment(environment) - if @opts[:environment] && (@opts[:environment] != environment.dirname) - logger.debug1(_("Only updating modules in environment %{opt_env} skipping environment %{env_path}") % {opt_env: @opts[:environment], env_path: environment.path}) + requested_envs = @settings.dig(:overrides, :environments, :requested_environments) + if !requested_envs.empty? && !requested_envs.include?(environment.dirname) + logger.debug1(_("Only updating modules in environment(s) %{opt_env} skipping environment %{env_path}") % {opt_env: requested_envs.inspect, env_path: environment.path}) else - logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @argv.inspect, env_path: environment.path}) - yield - end - end + logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @settings.dig(:overrides, :modules, :requested_modules).inspect, env_path: environment.path}) - def visit_puppetfile(puppetfile) - puppetfile.load - yield - end + updated_modules = environment.deploy - def visit_module(mod) - if @argv.include?(mod.name) - logger.info _("Deploying module %{mod_path}") % {mod_path: mod.path} - mod.sync(force: @force) - if mod.environment && @generate_types - logger.debug("Generating puppet types for environment '#{mod.environment.dirname}'...") - mod.environment.generate_types! + # We actually synced a module in this env + if !updated_modules.nil? && !updated_modules.empty? + # Record modified environment for postrun command + @modified_envs << environment.dirname + + if generate_types = @settings.dig(:overrides, :environments, :generate_types) + logger.debug("Generating puppet types for environment '#{environment.dirname}'...") + environment.generate_types! + end end - else - logger.debug1(_("Only updating modules %{modules}, skipping module %{mod_name}") % {modules: @argv.inspect, mod_name: mod.name}) end end def allowed_initialize_opts super.merge(environment: true, cachedir: :self, + 'exclude-spec': :self, 'no-force': :self, 'generate-types': :self, 'puppet-path': :self, - 'puppet-conf': :self) + 'puppet-conf': :self, + 'private-key': :self, + 'oauth-token': :self, + 'github-app-id': :self, + 'github-app-key': :self, + 'github-app-ttl': :self) end end end diff -Nru r10k-3.7.0/lib/r10k/action/puppetfile/check.rb r10k-4.0.0/lib/r10k/action/puppetfile/check.rb --- r10k-3.7.0/lib/r10k/action/puppetfile/check.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/puppetfile/check.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,6 +1,6 @@ -require 'r10k/puppetfile' require 'r10k/action/base' require 'r10k/errors/formatting' +require 'r10k/module_loader/puppetfile' module R10K module Action @@ -8,9 +8,20 @@ class Check < R10K::Action::Base def call - pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile) + options = { basedir: @root } + options[:overrides] = {} + options[:overrides][:modules] = { default_ref: @settings.dig(:git, :default_ref) } + options[:moduledir] = @moduledir if @moduledir + options[:puppetfile] = @puppetfile if @puppetfile + + loader = R10K::ModuleLoader::Puppetfile.new(**options) begin - pf.load! + loader.load! + loader.modules.each do |mod| + if mod.instance_of?(R10K::Module::Git) + mod.validate_ref_defined + end + end $stderr.puts _("Syntax OK") true rescue => e diff -Nru r10k-3.7.0/lib/r10k/action/puppetfile/install.rb r10k-4.0.0/lib/r10k/action/puppetfile/install.rb --- r10k-3.7.0/lib/r10k/action/puppetfile/install.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/puppetfile/install.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,7 +1,8 @@ -require 'r10k/puppetfile' -require 'r10k/errors/formatting' -require 'r10k/action/visitor' require 'r10k/action/base' +require 'r10k/content_synchronizer' +require 'r10k/errors/formatting' +require 'r10k/module_loader/puppetfile' +require 'r10k/util/cleaner' module R10K module Action @@ -9,35 +10,39 @@ class Install < R10K::Action::Base def call - @visit_ok = true - pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile, nil , @force) - pf.accept(self) - @visit_ok + begin + options = { basedir: @root, overrides: { force: @force || false } } + options[:overrides][:modules] = { default_ref: @settings.dig(:git, :default_ref) } + options[:moduledir] = @moduledir if @moduledir + options[:puppetfile] = @puppetfile if @puppetfile + options[:module_exclude_regex] = @module_exclude_regex if @module_exclude_regex + + loader = R10K::ModuleLoader::Puppetfile.new(**options) + loaded_content = loader.load! + + pool_size = @settings[:pool_size] || 4 + modules = loaded_content[:modules] + if pool_size > 1 + R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger) + else + R10K::ContentSynchronizer.serial_sync(modules, logger) + end + + R10K::Util::Cleaner.new(loaded_content[:managed_directories], + loaded_content[:desired_contents], + loaded_content[:purge_exclusions]).purge! + + true + rescue => e + logger.error R10K::Errors::Formatting.format_exception(e, @trace) + false + end end private - include R10K::Action::Visitor - - def visit_puppetfile(pf) - pf.load! - yield - pf.purge! - end - - def visit_module(mod) - @force ||= false - logger.info _("Updating module %{mod_path}") % {mod_path: mod.path} - - if mod.respond_to?(:desired_ref) && mod.desired_ref == :control_branch - logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." % {name: mod.name}) - end - - mod.sync(force: @force) - end - def allowed_initialize_opts - super.merge(root: :self, puppetfile: :self, moduledir: :self, force: :self ) + super.merge(root: :self, puppetfile: :self, moduledir: :self, :'module-exclude-regex' => :self, force: :self ) end end end diff -Nru r10k-3.7.0/lib/r10k/action/puppetfile/purge.rb r10k-4.0.0/lib/r10k/action/puppetfile/purge.rb --- r10k-3.7.0/lib/r10k/action/puppetfile/purge.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/puppetfile/purge.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,6 +1,7 @@ -require 'r10k/puppetfile' require 'r10k/action/base' require 'r10k/errors/formatting' +require 'r10k/module_loader/puppetfile' +require 'r10k/util/cleaner' module R10K module Action @@ -8,9 +9,17 @@ class Purge < R10K::Action::Base def call - pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile) - pf.load! - pf.purge! + options = { basedir: @root } + + options[:moduledir] = @moduledir if @moduledir + options[:puppetfile] = @puppetfile if @puppetfile + + loader = R10K::ModuleLoader::Puppetfile.new(**options) + loaded_content = loader.load! + R10K::Util::Cleaner.new(loaded_content[:managed_directories], + loaded_content[:desired_contents], + loaded_content[:purge_exclusions]).purge! + true rescue => e logger.error R10K::Errors::Formatting.format_exception(e, @trace) diff -Nru r10k-3.7.0/lib/r10k/action/runner.rb r10k-4.0.0/lib/r10k/action/runner.rb --- r10k-3.7.0/lib/r10k/action/runner.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/runner.rb 2023-08-01 15:06:51.000000000 +0000 @@ -44,10 +44,19 @@ overrides = {} overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir) - overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') - overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path') - overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil? - overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types') + if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') || @opts.key?(:'exclude-spec') || @opts.key?(:'puppet-conf') + overrides[:deploy] = {} + overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path') + overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] if @opts.key?(:'puppet-conf') + overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types') + overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec') + end + # If the log level has been given as an argument, ensure that output happens on stderr + if @opts.key?(:loglevel) + overrides[:logging] = {} + overrides[:logging][:level] = @opts[:loglevel] + overrides[:logging][:disable_default_stderr] = false + end with_overrides = config_settings.merge(overrides) do |key, oldval, newval| newval = oldval.merge(newval) if oldval.is_a? Hash @@ -55,6 +64,10 @@ newval end + # Credentials from the CLI override both the global and per-repo + # credentials from the config, and so need to be handled specially + with_overrides = add_credential_overrides(with_overrides) + @settings = R10K::Settings.global_settings.evaluate(with_overrides) R10K::Initializers::GlobalInitializer.new(@settings).call @@ -63,15 +76,20 @@ exit(8) end + # Set up authorization from license file if it wasn't + # already set via the config def setup_authorization - begin - license = R10K::Util::License.load - - if license.respond_to?(:authorization_token) - PuppetForge::Connection.authorization = license.authorization_token + if PuppetForge::Connection.authorization.nil? + begin + license = R10K::Util::License.load + + if license.respond_to?(:authorization_token) + logger.debug "Using token from license to connect to the Forge." + PuppetForge::Connection.authorization = license.authorization_token + end + rescue R10K::Error => e + logger.warn e.message end - rescue R10K::Error => e - logger.warn e.message end end @@ -92,6 +110,62 @@ results end + + def add_credential_overrides(overrides) + sshkey_path = @opts[:'private-key'] + token_path = @opts[:'oauth-token'] + app_id = @opts[:'github-app-id'] + app_private_key_path = @opts[:'github-app-key'] + app_ttl = @opts[:'github-app-ttl'] + + if sshkey_path && token_path + raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy." + end + + if sshkey_path && (app_private_key_path || app_id) + raise R10K::Error, "Cannot specify both an SSH key and an SSL key or Github App id to use with this deploy." + end + + if token_path && (app_private_key_path || app_id) + raise R10K::Error, "Cannot specify both an OAuth token and an SSL key or Github App id to use with this deploy." + end + + if app_id && ! app_private_key_path || app_private_key_path && ! app_id + raise R10K::Error, "Must specify both id and SSL private key to use Github App for this deploy." + end + + if sshkey_path + overrides[:git] ||= {} + overrides[:git][:private_key] = sshkey_path + if repo_settings = overrides[:git][:repositories] + repo_settings.each do |repo| + repo[:private_key] = sshkey_path + end + end + elsif token_path + overrides[:git] ||= {} + overrides[:git][:oauth_token] = token_path + if repo_settings = overrides[:git][:repositories] + repo_settings.each do |repo| + repo[:oauth_token] = token_path + end + end + elsif app_id + overrides[:git] ||= {} + overrides[:git][:github_app_id] = app_id + overrides[:git][:github_app_key] = app_private_key_path + overrides[:git][:github_app_ttl] = app_ttl + if repo_settings = overrides[:git][:repositories] + repo_settings.each do |repo| + repo[:github_app_id] = app_id + repo[:github_app_key] = app_private_key_path + repo[:github_app_ttl] = app_ttl + end + end + end + + overrides + end end end end diff -Nru r10k-3.7.0/lib/r10k/action/visitor.rb r10k-4.0.0/lib/r10k/action/visitor.rb --- r10k-3.7.0/lib/r10k/action/visitor.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/action/visitor.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,4 +1,5 @@ require 'r10k/errors/formatting' +require 'r10k/logging' module R10K module Action @@ -13,6 +14,8 @@ # @api private module Visitor + include R10K::Logging + # Dispatch to the type specific visitor method # # @param type [Symbol] The object type to dispatch for diff -Nru r10k-3.7.0/lib/r10k/cli/deploy.rb r10k-4.0.0/lib/r10k/cli/deploy.rb --- r10k-3.7.0/lib/r10k/cli/deploy.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/cli/deploy.rb 2023-08-01 15:06:51.000000000 +0000 @@ -10,6 +10,19 @@ module R10K::CLI module Deploy + + class TransformExcludeSpec + def call(input) + # To be backward compatible with the 3.x flag version of this setting, + # r10k allows this flag to have an optional argument. When no argument + # is supplied, cri defaults to setting the class to true, so we check + # for TrueClass here as well as "true". + return true if input == true || input == 'true' + return false if input == 'false' + raise ArgumentError + end + end + def self.command @cmd ||= Cri::Command.define do name 'deploy' @@ -21,9 +34,11 @@ (https://puppet.com/docs/puppet/latest/environments_about.html). DESCRIPTION - required nil, :cachedir, 'Specify a cachedir, overriding the value in config' + option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required flag nil, :'no-force', 'Prevent the overwriting of local module modifications' flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment' + option nil, :'exclude-spec', 'Exclude the module\'s spec dir for deployment', argument: :optional, + transform: TransformExcludeSpec.new option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd| unless File.executable? value $stderr.puts "The specified puppet executable #{value} is not executable." @@ -32,6 +47,11 @@ end end option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required + option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required + option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required + option nil, :'github-app-id', 'Github App id. Only valid with rugged provider', argument: :required + option nil, :'github-app-key', 'Github App private key. Only valid with rugged provider', argument: :required + option nil, :'github-app-ttl', 'Github App token expiration, in seconds. Only valid with rugged provider', default: "120", argument: :optional run do |opts, args, cmd| puts cmd.help(:verbose => opts[:verbose]) @@ -53,7 +73,7 @@ Environments can provide a Puppetfile at the root of the directory to deploy independent Puppet modules. To recursively deploy an environment, pass the -`--puppetfile` flag to the command. +`--modules` flag to the command. **NOTE**: If an environment has a Puppetfile when it is instantiated a recursive update will be forced. It is assumed that environments are dependent @@ -61,8 +81,11 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off. DESCRIPTION - flag :p, :puppetfile, 'Deploy modules from a puppetfile' - required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile' + flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)' + flag :m, :modules, 'Deploy modules' + flag nil, :incremental, 'Used with the --modules flag, only update those modules whose definition has changed or whose definition allows the version to float' + option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile', + argument: :required runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment) end @@ -82,7 +105,7 @@ try to deploy the given module names in all environments. DESCRIPTION - required :e, :environment, 'Update the modules in the given environment' + option :e, :environment, 'Update the modules in the given environment', argument: :required runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Module) end @@ -97,10 +120,12 @@ usage 'display' summary 'Display environments and modules in the deployment' - flag :p, :puppetfile, 'Display Puppetfile modules' + flag :p, :puppetfile, 'Display modules (deprecated, use -m)' + flag :m, :modules, 'Display modules' flag nil, :detail, 'Display detailed information' flag nil, :fetch, 'Update available environment lists from all remote sources' - required nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml' + option nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml', + argument: :required runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Display) end diff -Nru r10k-3.7.0/lib/r10k/cli/puppetfile.rb r10k-4.0.0/lib/r10k/cli/puppetfile.rb --- r10k-3.7.0/lib/r10k/cli/puppetfile.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/cli/puppetfile.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,5 +1,4 @@ require 'r10k/cli' -require 'r10k/puppetfile' require 'r10k/action/puppetfile' require 'cri' @@ -30,8 +29,9 @@ name 'install' usage 'install' summary 'Install all modules from a Puppetfile' - required nil, :moduledir, 'Path to install modules to' - required nil, :puppetfile, 'Path to puppetfile' + option nil, :moduledir, 'Path to install modules to', argument: :required + option nil, :puppetfile, 'Path to puppetfile', argument: :required + option nil, :'module-exclude-regex', 'A regex to exclude modules from installation. Helpful in CI environments.', argument: :required flag nil, :force, 'Force locally changed files to be overwritten' runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Install) end @@ -45,7 +45,7 @@ usage 'check' summary 'Try and load the Puppetfile to verify the syntax is correct.' - required nil, :puppetfile, 'Path to Puppetfile' + option nil, :puppetfile, 'Path to Puppetfile', argument: :required runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Check) end end @@ -58,8 +58,8 @@ usage 'purge' summary 'Purge unmanaged modules from a Puppetfile managed directory' - required nil, :moduledir, 'Path to install modules to' - required nil, :puppetfile, 'Path to Puppetfile' + option nil, :moduledir, 'Path to install modules to', argument: :required + option nil, :puppetfile, 'Path to Puppetfile', argument: :required runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Purge) end end diff -Nru r10k-3.7.0/lib/r10k/content_synchronizer.rb r10k-4.0.0/lib/r10k/content_synchronizer.rb --- r10k-3.7.0/lib/r10k/content_synchronizer.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/content_synchronizer.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,100 @@ +module R10K + module ContentSynchronizer + + def self.serial_accept(modules, visitor, loader) + visitor.visit(:puppetfile, loader) do + serial_sync(modules) + end + end + + def self.serial_sync(modules) + updated_modules = [] + modules.each do |mod| + updated = mod.sync + updated_modules << mod.name if updated + end + updated_modules + end + + # Returns a Queue of the names of modules actually updated + def self.concurrent_accept(modules, visitor, loader, pool_size, logger) + mods_queue = modules_visit_queue(modules, visitor, loader) + sync_queue(mods_queue, pool_size, logger) + end + + # Returns a Queue of the names of modules actually updated + def self.concurrent_sync(modules, pool_size, logger) + mods_queue = modules_sync_queue(modules) + sync_queue(mods_queue, pool_size, logger) + end + + # Returns a Queue of the names of modules actually updated + def self.sync_queue(mods_queue, pool_size, logger) + logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size} + updated_modules = Queue.new + thread_pool = pool_size.times.map { sync_thread(mods_queue, logger, updated_modules) } + thread_exception = nil + + # If any threads raise an exception the deployment is considered a failure. + # In that event clear the queue, wait for other threads to finish their + # current work, then re-raise the first exception caught. + begin + thread_pool.each(&:join) + # Return the list of all modules that were actually updated + updated_modules + rescue => e + logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message} + mods_queue.clear + thread_exception ||= e + retry + ensure + raise thread_exception unless thread_exception.nil? + end + end + + def self.modules_visit_queue(modules, visitor, loader) + Queue.new.tap do |queue| + visitor.visit(:puppetfile, loader) do + enqueue_modules(queue, modules) + end + end + end + + def self.modules_sync_queue(modules) + Queue.new.tap do |queue| + enqueue_modules(queue, modules) + end + end + + def self.enqueue_modules(queue, modules) + modules_by_cachedir = modules.group_by { |mod| mod.cachedir } + modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || [] + + modules_without_vcs_cachedir.each {|mod| queue << Array(mod) } + modules_by_cachedir.values.each {|mods| queue << mods } + end + + def self.sync_thread(mods_queue, logger, updated_modules) + Thread.new do + begin + while mods = mods_queue.pop(true) do + mods.each do |mod| + begin + updated = mod.sync + updated_modules << mod.name if updated + rescue Exception => e + logger.error _("Module %{mod_name} failed to synchronize due to %{message}") % {mod_name: mod.name, message: e.message} + raise e + end + end + end + rescue ThreadError => e + logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id} + Thread.exit + rescue => e + Thread.main.raise(e) + end + end + end + end +end diff -Nru r10k-3.7.0/lib/r10k/deployment.rb r10k-4.0.0/lib/r10k/deployment.rb --- r10k-3.7.0/lib/r10k/deployment.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/deployment.rb 2023-08-01 15:06:51.000000000 +0000 @@ -118,7 +118,7 @@ raise R10K::Error, _("Unable to load sources; the supplied configuration does not define the 'sources' key") end @_sources = sources.map do |(name, hash)| - R10K::Source.from_hash(name, hash) + R10K::Source.from_hash(name, hash.merge({overrides: @config[:overrides]})) end end diff -Nru r10k-3.7.0/lib/r10k/environment/bare.rb r10k-4.0.0/lib/r10k/environment/bare.rb --- r10k-3.7.0/lib/r10k/environment/bare.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/bare.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -class R10K::Environment::Bare < R10K::Environment::WithModules - - R10K::Environment.register(:bare, self) - - def sync - path.mkpath - end - - def status - :not_applicable - end - - def signature - 'bare-default' - end -end diff -Nru r10k-3.7.0/lib/r10k/environment/base.rb r10k-4.0.0/lib/r10k/environment/base.rb --- r10k-3.7.0/lib/r10k/environment/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/base.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,3 +1,7 @@ +require 'r10k/content_synchronizer' +require 'r10k/logging' +require 'r10k/module_loader/puppetfile' +require 'r10k/util/cleaner' require 'r10k/util/subprocess' # This class defines a common interface for environment implementations. @@ -5,6 +9,8 @@ # @since 1.3.0 class R10K::Environment::Base + include R10K::Logging + # @!attribute [r] name # @return [String] A name for this environment that is unique to the given source attr_reader :name @@ -31,6 +37,10 @@ # @return [String] The puppetfile name (relative) attr_reader :puppetfile_name + attr_reader :managed_directories, :desired_contents + + attr_reader :loader + # Initialize the given environment. # # @param name [String] The unique name describing this environment. @@ -43,13 +53,31 @@ @basedir = basedir @dirname = dirname @options = options - @puppetfile_name = options[:puppetfile_name] + @puppetfile_name = options.delete(:puppetfile_name) + @overrides = options.delete(:overrides) || {} @full_path = File.join(@basedir, @dirname) @path = Pathname.new(File.join(@basedir, @dirname)) - @puppetfile = R10K::Puppetfile.new(@full_path, nil, nil, @puppetfile_name) + @puppetfile = R10K::Puppetfile.new(@full_path, + {overrides: @overrides, + force: @overrides.dig(:modules, :force), + puppetfile_name: @puppetfile_name}) @puppetfile.environment = self + + loader_options = { basedir: @full_path, overrides: @overrides, environment: self } + loader_options[:puppetfile] = @puppetfile_name if @puppetfile_name + + @loader = R10K::ModuleLoader::Puppetfile.new(**loader_options) + + if @overrides.dig(:environments, :incremental) + @loader.load_metadata + end + + @base_modules = nil + @purge_exclusions = nil + @managed_directories = [ @full_path ] + @desired_contents = [] end # Synchronize the given environment. @@ -99,8 +127,18 @@ # @return [Array] All modules defined in the Puppetfile # associated with this environment. def modules - @puppetfile.load - @puppetfile.modules + if @base_modules.nil? + load_puppetfile_modules + end + + @base_modules + end + + # @return [Array] Whether or not the given module + # conflicts with any modules already defined in the r10k environment + # object. + def module_conflicts?(mod) + false end def accept(visitor) @@ -109,16 +147,50 @@ end end + + # Returns a Queue of the names of modules actually updated + def deploy + if @base_modules.nil? + load_puppetfile_modules + end + + if ! @base_modules.empty? + pool_size = @overrides.dig(:modules, :pool_size) + updated_modules = R10K::ContentSynchronizer.concurrent_sync(@base_modules, pool_size, logger) + end + + if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile) + logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...") + @puppetfile_cleaner.purge! + end + + updated_modules + end + + def load_puppetfile_modules + loaded_content = @loader.load + @base_modules = loaded_content[:modules] + + @purge_exclusions = determine_purge_exclusions(loaded_content[:managed_directories], + loaded_content[:desired_contents]) + + @puppetfile_cleaner = R10K::Util::Cleaner.new(loaded_content[:managed_directories], + loaded_content[:desired_contents], + loaded_content[:purge_exclusions]) + end + def whitelist(user_whitelist=[]) user_whitelist.collect { |pattern| File.join(@full_path, pattern) } end - def purge_exclusions + def determine_purge_exclusions(pf_managed_dirs = @puppetfile.managed_directories, + pf_desired_contents = @puppetfile.desired_contents) + list = [File.join(@full_path, '.r10k-deploy.json')].to_set - list += @puppetfile.managed_directories + list += pf_managed_dirs - list += @puppetfile.desired_contents.flat_map do |item| + list += pf_desired_contents.flat_map do |item| desired_tree = [] if File.directory?(item) @@ -136,6 +208,14 @@ list.to_a end + def purge_exclusions + if @purge_exclusions.nil? + load_puppetfile_modules + end + + @purge_exclusions + end + def generate_types! argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf] subproc = R10K::Util::Subprocess.new(argv) diff -Nru r10k-3.7.0/lib/r10k/environment/git.rb r10k-4.0.0/lib/r10k/environment/git.rb --- r10k-3.7.0/lib/r10k/environment/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/git.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,4 +1,3 @@ -require 'r10k/logging' require 'r10k/puppetfile' require 'r10k/git/stateful_repository' require 'forwardable' @@ -8,8 +7,6 @@ # @since 1.3.0 class R10K::Environment::Git < R10K::Environment::WithModules - include R10K::Logging - R10K::Environment.register(:git, self) # Register git as the default environment type R10K::Environment.register(nil, self) @@ -27,6 +24,8 @@ # @return [R10K::Git::StatefulRepository] The git repo backing this environment attr_reader :repo + include R10K::Util::Setopts + # Initialize the given Git environment. # # @param name [String] The unique name describing this environment. @@ -38,8 +37,21 @@ # @param options [String] :ref The git reference to use for this environment def initialize(name, basedir, dirname, options = {}) super - @remote = options[:remote] - @ref = options[:ref] + setopts(options, { + # Standard option interface + :version => :ref, + :source => :remote, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :ref => :self, + :remote => :self, + + }, raise_on_unhandled: false) + # TODO: in r10k 4.0.0, a major version bump, stop allowing garbage options. + # We only allow them now, here, on this object, because prior to adopting + # setopts in the constructor, this object type didn't do any validation + # checking of options passed, and would permit garbage parameters. @repo = R10K::Git::StatefulRepository.new(@remote, @basedir, @dirname) end diff -Nru r10k-3.7.0/lib/r10k/environment/name.rb r10k-4.0.0/lib/r10k/environment/name.rb --- r10k-3.7.0/lib/r10k/environment/name.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/name.rb 2023-08-01 15:06:51.000000000 +0000 @@ -6,19 +6,24 @@ class Name # @!attribute [r] name - # @return [String] The unmodified name of the environment + # @return [String] The functional name of the environment derived from inputs and options. attr_reader :name - INVALID_CHARACTERS = %r[\W] + # @!attribute [r] original_name + # @return [String] The unmodified name originally given to create the object. + attr_reader :original_name - def initialize(name, opts) - @name = name - @opts = opts + INVALID_CHARACTERS = %r[\W] + def initialize(original_name, opts) @source = opts[:source] @prefix = opts[:prefix] @invalid = opts[:invalid] + @name = derive_name(original_name, opts[:strip_component]) + @original_name = original_name + @opts = opts + case @invalid when 'correct_and_warn' @validate = true @@ -71,8 +76,26 @@ private - def derive_prefix(source,prefix) + def derive_name(original_name, strip_component) + return original_name unless strip_component + + unless strip_component.is_a?(String) + raise _('Improper configuration value given for strip_component setting in %{src} source. ' \ + 'Value must be a string, a /regex/, false, or omitted. Got "%{val}" (%{type})' \ + % {src: @source, val: strip_component, type: strip_component.class}) + end + if %r{^/.*/$}.match(strip_component) + regex = Regexp.new(strip_component[1..-2]) + original_name.gsub(regex, '') + elsif original_name.start_with?(strip_component) + original_name[strip_component.size..-1] + else + original_name + end + end + + def derive_prefix(source,prefix) if prefix == true "#{source}_" elsif prefix.is_a? String diff -Nru r10k-3.7.0/lib/r10k/environment/plain.rb r10k-4.0.0/lib/r10k/environment/plain.rb --- r10k-3.7.0/lib/r10k/environment/plain.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/plain.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,16 @@ +class R10K::Environment::Plain < R10K::Environment::WithModules + + R10K::Environment.register(:plain, self) + + def sync + path.mkpath + end + + def status + :not_applicable + end + + def signature + 'plain-default' + end +end diff -Nru r10k-3.7.0/lib/r10k/environment/svn.rb r10k-4.0.0/lib/r10k/environment/svn.rb --- r10k-3.7.0/lib/r10k/environment/svn.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/svn.rb 2023-08-01 15:06:51.000000000 +0000 @@ -7,8 +7,6 @@ # @since 1.3.0 class R10K::Environment::SVN < R10K::Environment::Base - include R10K::Logging - R10K::Environment.register(:svn, self) # @!attribute [r] remote @@ -44,8 +42,17 @@ # @option options [String] :password The SVN password def initialize(name, basedir, dirname, options = {}) super - - setopts(options, {:remote => :self, :username => :self, :password => :self, :puppetfile_name => :self }) + setopts(options, { + # Standard option interface + :source => :remote, + :version => :expected_revision, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :remote => :self, + :username => :self, + :password => :self, + }) @working_dir = R10K::SVN::WorkingDir.new(Pathname.new(@full_path), :username => @username, :password => @password) end @@ -61,7 +68,7 @@ if @working_dir.is_svn? @working_dir.update else - @working_dir.checkout(@remote) + @working_dir.checkout(@remote, @expected_revision) end @synced = true end diff -Nru r10k-3.7.0/lib/r10k/environment/tarball.rb r10k-4.0.0/lib/r10k/environment/tarball.rb --- r10k-3.7.0/lib/r10k/environment/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/tarball.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,78 @@ +require 'r10k/util/setopts' +require 'r10k/tarball' +require 'r10k/environment' + +class R10K::Environment::Tarball < R10K::Environment::WithModules + + R10K::Environment.register(:tarball, self) + + # @!attribute [r] tarball + # @api private + # @return [R10K::Tarball] + attr_reader :tarball + + include R10K::Util::Setopts + + # Initialize the given tarball environment. + # + # @param name [String] The unique name describing this environment. + # @param basedir [String] The base directory where this environment will be created. + # @param dirname [String] The directory name for this environment. + # @param options [Hash] An additional set of options for this environment. + # + # @param options [String] :source Where to get the tarball from + # @param options [String] :version The sha256 digest of the tarball + def initialize(name, basedir, dirname, options = {}) + super + setopts(options, { + # Standard option interface + :type => ::R10K::Util::Setopts::Ignore, + :source => :self, + :version => :checksum, + + # Type-specific options + :checksum => :self, + }) + + @tarball = R10K::Tarball.new(name, @source, checksum: @checksum) + end + + def path + @path ||= Pathname.new(File.join(@basedir, @dirname)) + end + + def sync + tarball.get unless tarball.cache_valid? + case status + when :absent, :mismatched + tarball.unpack(path.to_s) + # Untracked files left behind from previous extractions are expected to + # be deleted by r10k's purge facility. + end + end + + def status + if not path.exist? + :absent + elsif not (tarball.cache_valid? && tarball.insync?(path.to_s, ignore_untracked_files: true)) + :mismatched + else + :insync + end + end + + def signature + @checksum || @tarball.cache_checksum + end + + include R10K::Util::Purgeable + + # Returns an array of the full paths to all the content being managed. + # @note This implements a required method for the Purgeable mixin + # @return [Array] + def desired_contents + desired = [] + desired += @tarball.paths.map { |entry| File.join(@full_path, entry) } + desired += super + end +end diff -Nru r10k-3.7.0/lib/r10k/environment/with_modules.rb r10k-4.0.0/lib/r10k/environment/with_modules.rb --- r10k-3.7.0/lib/r10k/environment/with_modules.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment/with_modules.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,4 +1,3 @@ -require 'r10k/logging' require 'r10k/util/purgeable' # This abstract base class implements an environment that can include module @@ -7,8 +6,6 @@ # @since 3.4.0 class R10K::Environment::WithModules < R10K::Environment::Base - include R10K::Logging - # @!attribute [r] moduledir # @return [String] The directory to install environment-defined modules # into (default: #{basedir}/modules) @@ -24,8 +21,9 @@ # @param options [String] :moduledir The path to install modules to # @param options [Hash] :modules Modules to add to the environment def initialize(name, basedir, dirname, options = {}) - super(name, basedir, dirname, options) + super + @all_modules = nil @managed_content = {} @modules = [] @moduledir = case options[:moduledir] @@ -46,39 +44,97 @@ # - The r10k environment object # - A Puppetfile in the environment's content def modules - return @modules if @puppetfile.nil? + if @all_modules.nil? + puppetfile_modules = super() + @all_modules = @modules + puppetfile_modules + end + + @all_modules + end + + def module_conflicts?(mod_b) + conflict = @modules.any? { |mod_a| mod_a.name == mod_b.name } + return false unless conflict + + msg_vars = {src: mod_b.origin, name: mod_b.name} + msg_error = _('Environment and %{src} both define the "%{name}" module' % msg_vars) + msg_continue = _("#{msg_error}. The %{src} definition will be ignored" % msg_vars) + + case conflict_opt = @options[:module_conflicts] + when 'override_and_warn', nil + logger.warn msg_continue + when 'override' + logger.debug msg_continue + when 'error' + raise R10K::Error, msg_error + else + raise R10K::Error, _('Unexpected value for `module_conflicts` setting in %{env} ' \ + 'environment: %{val}' % {env: self.name, val: conflict_opt}) + end - @puppetfile.load unless @puppetfile.loaded? - @modules + @puppetfile.modules + true end def accept(visitor) visitor.visit(:environment, self) do @modules.each do |mod| - mod.accept(visitor) + mod.sync end puppetfile.accept(visitor) - validate_no_module_conflicts end end + def deploy + @modules.each do |mod| + mod.sync + end + + super + end + def load_modules(module_hash) module_hash.each do |name, args| + if !args.is_a?(Hash) + args = { type: 'forge', version: args } + end + add_module(name, args) end end + def resolve_path(base, dirname, path) + if Pathname.new(path).absolute? + cleanpath(path) + else + cleanpath(File.join(base, dirname, path)) + end + end + + # .cleanpath is as good as we can do without touching the filesystem. + # The .realpath methods will choke if some of the intermediate paths + # are missing, even though in some cases we will create them later as + # needed. + def cleanpath(path) + Pathname.new(path).cleanpath.to_s + end + + def validate_install_path(path, modname) + unless /^#{Regexp.escape(@basedir)}.*/ =~ path + raise R10K::Error.new("Environment cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{@basedir}") + end + true + end + # @param [String] name - # @param [*Object] args + # @param [Hash] args def add_module(name, args) - if args.is_a?(Hash) - # symbolize keys in the args hash - args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo } - end + # symbolize keys in the args hash + args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo } + args[:overrides] = @overrides - if args.is_a?(Hash) && install_path = args.delete(:install_path) - install_path = resolve_install_path(install_path) + if install_path = args.delete(:install_path) + install_path = resolve_path(@basedir, @dirname, install_path) validate_install_path(install_path, name) else install_path = @moduledir @@ -88,35 +144,14 @@ @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path) mod = R10K::Module.new(name, install_path, args, self.name) - mod.origin = 'Environment' + mod.origin = :environment @managed_content[install_path] << mod.name @modules << mod end - def validate_no_module_conflicts - @puppetfile.load unless @puppetfile.loaded? - conflicts = (@modules + @puppetfile.modules) - .group_by { |mod| mod.name } - .select { |_, v| v.size > 1 } - .map(&:first) - unless conflicts.empty? - msg = _('Puppetfile cannot contain module names defined by environment %{name}') % {name: self.name} - msg += ' ' - msg += _("Remove the conflicting definitions of the following modules: %{conflicts}" % { conflicts: conflicts.join(' ') }) - raise R10K::Error.new(msg) - end - end - include R10K::Util::Purgeable - # Returns an array of the full paths that can be purged. - # @note This implements a required method for the Purgeable mixin - # @return [Array] - def managed_directories - [@full_path] - end - # Returns an array of the full paths of filenames that should exist. Files # inside managed_directories that are not listed in desired_contents will # be purged. diff -Nru r10k-3.7.0/lib/r10k/environment.rb r10k-4.0.0/lib/r10k/environment.rb --- r10k-3.7.0/lib/r10k/environment.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/environment.rb 2023-08-01 15:06:51.000000000 +0000 @@ -30,8 +30,9 @@ require 'r10k/environment/base' require 'r10k/environment/with_modules' - require 'r10k/environment/bare' + require 'r10k/environment/plain' require 'r10k/environment/git' require 'r10k/environment/svn' + require 'r10k/environment/tarball' end end diff -Nru r10k-3.7.0/lib/r10k/errors.rb r10k-4.0.0/lib/r10k/errors.rb --- r10k-3.7.0/lib/r10k/errors.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/errors.rb 2023-08-01 15:06:51.000000000 +0000 @@ -58,4 +58,9 @@ str.gsub(/^/, prefix) end end + + # An error class for configuration errors + # + class ConfigError < Error + end end diff -Nru r10k-3.7.0/lib/r10k/forge/module_release.rb r10k-4.0.0/lib/r10k/forge/module_release.rb --- r10k-3.7.0/lib/r10k/forge/module_release.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/forge/module_release.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,5 +1,6 @@ require 'r10k/logging' require 'r10k/settings/mixin' +require 'r10k/util/cacheable' require 'fileutils' require 'tmpdir' require 'puppet_forge' @@ -13,7 +14,7 @@ def_setting_attr :proxy def_setting_attr :baseurl - def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/cache': '/root/.r10k/cache') + def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir include R10K::Logging diff -Nru r10k-3.7.0/lib/r10k/git/cache.rb r10k-4.0.0/lib/r10k/git/cache.rb --- r10k-3.7.0/lib/r10k/git/cache.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/cache.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,6 +3,7 @@ require 'r10k/settings' require 'r10k/instance_cache' require 'forwardable' +require 'r10k/util/cacheable' # Cache Git repository mirrors for object database reuse. # @@ -15,18 +16,9 @@ class R10K::Git::Cache include R10K::Settings::Mixin + include R10K::Util::Cacheable - #@api private - def self.determine_cache_root - if R10K::Util::Platform.windows? - File.join(ENV['LOCALAPPDATA'], 'r10k', 'git') - else - File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git') - end - end - private_class_method :determine_cache_root - - def_setting_attr :cache_root, determine_cache_root + def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir('git') @instance_cache = R10K::InstanceCache.new(self) @@ -109,8 +101,7 @@ alias cached? exist? - # Reformat the remote name into something that can be used as a directory def sanitized_dirname - @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-') + @sanitized_dirname ||= super(@remote) end end diff -Nru r10k-3.7.0/lib/r10k/git/rugged/bare_repository.rb r10k-4.0.0/lib/r10k/git/rugged/bare_repository.rb --- r10k-3.7.0/lib/r10k/git/rugged/bare_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/rugged/bare_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -56,15 +56,16 @@ logger.warn { _("Rugged versions prior to 0.24.0 do not support pruning stale branches during fetch, please upgrade your \'rugged\' gem. (Current version is: %{version})") % {version: Rugged::Version} } end - options = {:credentials => credentials, :prune => true} - refspecs = ['+refs/*:refs/*'] - remote = remotes[remote_name] proxy = R10K::Git.get_proxy_for_remote(remote) + + options = {:credentials => credentials, :prune => true, :proxy_url => proxy} + refspecs = ['+refs/*:refs/*'] + results = nil R10K::Git.with_proxy(proxy) do - results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } + results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } end report_transfer(results, remote_name) diff -Nru r10k-3.7.0/lib/r10k/git/rugged/base_repository.rb r10k-4.0.0/lib/r10k/git/rugged/base_repository.rb --- r10k-3.7.0/lib/r10k/git/rugged/base_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/rugged/base_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -20,7 +20,8 @@ else object.oid end - rescue ::Rugged::ReferenceError + rescue ::Rugged::ReferenceError, ::Rugged::OdbError => e + logger.debug2(_("Unable to resolve %{pattern}: %{e} ") % {pattern: pattern, e: e }) nil end @@ -60,6 +61,16 @@ remotes_hash end + # Update a remote URL + # @param [String] The remote URL of the git repository + # @param [String] An optional remote name for the git repository + def update_remote(remote, remote_name='origin') + if @_rugged_repo + logger.debug2(_("Remote URL is different from cache, updating %{orig} to %{update}") % {orig: remotes[remote_name], update: remote}) + @_rugged_repo.remotes.set_url(remote_name, remote) + end + end + private def with_repo(opts={}) diff -Nru r10k-3.7.0/lib/r10k/git/rugged/cache.rb r10k-4.0.0/lib/r10k/git/rugged/cache.rb --- r10k-3.7.0/lib/r10k/git/rugged/cache.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/rugged/cache.rb 2023-08-01 15:06:51.000000000 +0000 @@ -8,4 +8,12 @@ def self.bare_repository R10K::Git::Rugged::BareRepository end + + # Update the remote URL if the cache differs from the current configuration + def sync! + if cached? && @repo.remotes['origin'] != @remote + @repo.update_remote(@remote) + end + super + end end diff -Nru r10k-3.7.0/lib/r10k/git/rugged/credentials.rb r10k-4.0.0/lib/r10k/git/rugged/credentials.rb --- r10k-3.7.0/lib/r10k/git/rugged/credentials.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/rugged/credentials.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,6 +1,10 @@ require 'r10k/git/rugged' require 'r10k/git/errors' require 'r10k/logging' +require 'json' +require 'jwt' +require 'net/http' +require 'openssl' # Generate credentials for secured remote connections. # @@ -61,11 +65,62 @@ end def get_plaintext_credentials(url, username_from_url) - user = get_git_username(url, username_from_url) - password = URI.parse(url).password || '' + per_repo_oauth_token = nil + per_repo_github_app_id = nil + per_repo_github_app_key = nil + per_repo_github_app_ttl = nil + + if per_repo_settings = R10K::Git.get_repo_settings(url) + per_repo_oauth_token = per_repo_settings[:oauth_token] + per_repo_github_app_id = per_repo_settings[:github_app_id] + per_repo_github_app_key = per_repo_settings[:github_app_key] + per_repo_github_app_ttl = per_repo_settings[:github_app_ttl] + end + + app_id = per_repo_github_app_id || R10K::Git.settings[:github_app_id] + app_key = per_repo_github_app_key || R10K::Git.settings[:github_app_key] + app_ttl = per_repo_github_app_ttl || R10K::Git.settings[:github_app_ttl] + + if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token] + @oauth_token ||= extract_token(token_path, url) + + user = 'x-oauth-token' + password = @oauth_token + elsif app_id && app_key && app_ttl + user = 'x-access-token' + password = github_app_token(app_id, app_key, app_ttl) + else + user = get_git_username(url, username_from_url) + password = URI.parse(url).password || '' + end Rugged::Credentials::UserPassword.new(username: user, password: password) end + def extract_token(token_path, url) + if token_path == '-' + token = $stdin.read.strip + logger.debug2 _("Using OAuth token from stdin for URL %{url}") % { url: url } + elsif File.readable?(token_path) + token = File.read(token_path).strip + logger.debug2 _("Using OAuth token from %{token_path} for URL %{url}") % { token_path: token_path, url: url } + else + raise R10K::Git::GitError, _("%{path} is missing or unreadable, cannot load OAuth token") % { path: token_path } + end + + unless valid_token?(token) + raise R10K::Git::GitError, _("Supplied OAuth token contains invalid characters.") + end + + token + end + + # This regex is the only real requirement for OAuth token format, + # per https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/ + # Bitbucket's tokens also can include an underscore, so that is added here. + def valid_token?(token) + return token =~ /^[\w\-\.~_\+\/]+$/ + end + def get_default_credentials(url, username_from_url) Rugged::Credentials::Default.new end @@ -75,7 +130,7 @@ user = nil - if !username_from_url.nil? + if !username_from_url.nil? && !username_from_url.empty? user = username_from_url logger.debug2 _("URL %{url} includes the username %{username}, using that user for authentication.") % {url: url.inspect, username: username_from_url} elsif git_user @@ -88,4 +143,63 @@ user end + + def github_app_token(app_id, private_key, ttl) + raise R10K::Git::GitError, _('Github App id contains invalid characters.') unless app_id =~ /^\d+$/ + raise R10K::Git::GitError, _('Github App token ttl contains invalid characters.') unless ttl =~ /^\d+$/ + raise R10K::Git::GitError, _('Github App key is missing or unreadable') unless File.readable?(private_key) + + begin + ssl_key = OpenSSL::PKey::RSA.new(File.read(private_key).strip) + unless ssl_key.private? + raise R10K::Git::GitError, _('Github App key is not a valid SSL private key') + end + rescue OpenSSL::PKey::RSAError + raise R10K::Git::GitError, _('Github App key is not a valid SSL key') + end + + logger.debug2 _("Using Github App id %{app_id} with SSL key from %{key_path}") % { key_path: private_key, app_id: app_id } + + jwt_issue_time = Time.now.to_i - 60 + jwt_exp_time = (jwt_issue_time + 60) + ttl.to_i + payload = { iat: jwt_issue_time, exp: jwt_exp_time, iss: app_id } + jwt = JWT.encode(payload, ssl_key, "RS256") + + get = URI.parse("https://api.github.com/app/installations") + get_request = Net::HTTP::Get.new(get) + get_request["Authorization"] = "Bearer #{jwt}" + get_request["Accept"] = "application/vnd.github.v3+json" + get_req_options = { use_ssl: get.scheme == "https", } + get_response = Net::HTTP.start(get.hostname, get.port, get_req_options) do |http| + http.request(get_request) + end + + unless (get_response.class < Net::HTTPSuccess) + logger.debug2 _("Unexpected response code: #{get_response.code}\nResponse body: #{get_response.body}") + raise R10K::Git::GitError, _("Error using private key to get Github App access token from url") + end + + access_tokens_url = JSON.parse(get_response.body)[0]['access_tokens_url'] + + post = URI.parse(access_tokens_url) + post_request = Net::HTTP::Post.new(post) + post_request["Authorization"] = "Bearer #{jwt}" + post_request["Accept"] = "application/vnd.github.v3+json" + post_req_options = { use_ssl: post.scheme == "https", } + post_response = Net::HTTP.start(post.hostname, post.port, post_req_options) do |http| + http.request(post_request) + end + + unless (post_response.class < Net::HTTPSuccess) + logger.debug2 _("Unexpected response code: #{post_response.code}\nResponse body: #{post_response.body}") + raise R10K::Git::GitError, _("Error using private key to generate access token from #{access_token_url}") + end + + token = JSON.parse(post_response.body)['token'] + + raise R10K::Git::GitError, _("Github App token contains invalid characters.") unless valid_token?(token) + + logger.debug2 _("Github App token generated, expires at: %{expire}") % {expire: JSON.parse(post_response.body)['expires_at']} + token + end end diff -Nru r10k-3.7.0/lib/r10k/git/rugged/thin_repository.rb r10k-4.0.0/lib/r10k/git/rugged/thin_repository.rb --- r10k-3.7.0/lib/r10k/git/rugged/thin_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/rugged/thin_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -75,6 +75,13 @@ end end + def stage_files(files=['.']) + with_repo do |repo| + index = repo.index + files.each { |p| index.add( :path => p ) } + end + end + private # Override the parent class repo setup so that we can make sure the alternates file is up to date diff -Nru r10k-3.7.0/lib/r10k/git/rugged/working_repository.rb r10k-4.0.0/lib/r10k/git/rugged/working_repository.rb --- r10k-3.7.0/lib/r10k/git/rugged/working_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/rugged/working_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -27,6 +27,7 @@ def clone(remote, opts = {}) logger.debug1 { _("Cloning '%{remote}' into %{path}") % {remote: remote, path: @path } } + proxy = R10K::Git.get_proxy_for_remote(remote) # libgit2/rugged doesn't support cloning a repository and providing an # alternate object database, making the handling of :alternates a noop. # Unfortunately this means that this method can't really use alternates @@ -34,10 +35,9 @@ # repository. However alternate databases can be handled when an existing # repository is loaded, so loading a cloned repo will correctly use # alternate object database. - options = {:credentials => credentials} + options = {:credentials => credentials, :proxy_url => proxy} options.merge!(:alternates => [File.join(opts[:reference], 'objects')]) if opts[:reference] - proxy = R10K::Git.get_proxy_for_remote(remote) R10K::Git.with_proxy(proxy) do @_rugged_repo = ::Rugged::Repository.clone_at(remote, @path.to_s, options) @@ -85,15 +85,17 @@ def fetch(remote_name = 'origin') logger.debug1 { _("Fetching remote '%{remote}' at %{path}") % {remote: remote_name, path: @path} } - options = {:credentials => credentials} - refspecs = ["+refs/heads/*:refs/remotes/#{remote_name}/*"] remote = remotes[remote_name] proxy = R10K::Git.get_proxy_for_remote(remote) + + options = {:credentials => credentials, :proxy_url => proxy} + refspecs = ["+refs/heads/*:refs/remotes/#{remote_name}/*"] + results = nil R10K::Git.with_proxy(proxy) do - results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) } + results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) } end report_transfer(results, remote) @@ -117,11 +119,15 @@ with_repo { |repo| repo.config['remote.origin.url'] } end - def dirty? + def dirty?(exclude_spec=true) with_repo do |repo| - diff = repo.diff_workdir('HEAD') + if exclude_spec + diff = repo.diff_workdir('HEAD').select { |d| ! d.delta.old_file[:path].start_with?('spec/') } + else + diff = repo.diff_workdir('HEAD').to_a + end - diff.each_patch do |p| + diff.each do |p| logger.debug(_("Found local modifications in %{file_path}" % {file_path: File.join(@path, p.delta.old_file[:path])})) logger.debug1(p.to_s) end diff -Nru r10k-3.7.0/lib/r10k/git/shellgit/thin_repository.rb r10k-4.0.0/lib/r10k/git/shellgit/thin_repository.rb --- r10k-3.7.0/lib/r10k/git/shellgit/thin_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/shellgit/thin_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -43,6 +43,10 @@ git(['ls-tree', '-t', '-r', '--name-only', ref], :path => @path.to_s).stdout.split("\n") end + def stage_files(files=['.']) + git(['add', files].flatten, :path => @path.to_s) + end + private def setup_cache_remote diff -Nru r10k-3.7.0/lib/r10k/git/shellgit/working_repository.rb r10k-4.0.0/lib/r10k/git/shellgit/working_repository.rb --- r10k-3.7.0/lib/r10k/git/shellgit/working_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/shellgit/working_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -90,11 +90,12 @@ end # does the working tree have local modifications to tracked files? - def dirty? + def dirty?(exclude_spec=true) result = git(['diff-index', '--exit-code', '--name-only', 'HEAD'], :path => @path.to_s, :raise_on_fail => false) if result.exit_code != 0 - dirty_files = result.stdout.split('\n') + dirty_files = result.stdout.split("\n") + dirty_files.delete_if { |f| f.start_with?('spec/') } if exclude_spec dirty_files.each do |file| logger.debug(_("Found local modifications in %{file_path}" % {file_path: File.join(@path, file)})) @@ -103,7 +104,7 @@ logger.debug1 { git(['diff-index', '-p', 'HEAD', file], :path => @path.to_s, :raise_on_fail => false).stdout } end - return true + return dirty_files.size > 0 else return false end diff -Nru r10k-3.7.0/lib/r10k/git/stateful_repository.rb r10k-4.0.0/lib/r10k/git/stateful_repository.rb --- r10k-3.7.0/lib/r10k/git/stateful_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git/stateful_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -35,7 +35,8 @@ @cache.resolve(ref) end - def sync(ref, force=true) + # Returns true if the sync actually updated the repo, false otherwise + def sync(ref, force=true, exclude_spec=true) @cache.sync if sync_cache?(ref) sha = @cache.resolve(ref) @@ -44,8 +45,9 @@ raise R10K::Git::UnresolvableRefError.new(_("Unable to sync repo to unresolvable ref '%{ref}'") % {ref: ref}, :git_dir => @repo.git_dir) end - workdir_status = status(ref) + workdir_status = status(ref, exclude_spec) + updated = true case workdir_status when :absent logger.debug(_("Cloning %{repo_path} and checking out %{ref}") % {repo_path: @repo.path, ref: ref }) @@ -64,22 +66,29 @@ @repo.checkout(sha, {:force => force}) else logger.warn(_("Skipping %{repo_path} due to local modifications") % {repo_path: @repo.path}) + updated = false end else logger.debug(_("%{repo_path} is already at Git ref %{ref}") % {repo_path: @repo.path, ref: ref }) + updated = false end + updated end - def status(ref) + def status(ref, exclude_spec=true) if !@repo.exist? :absent + elsif !@cache.exist? + :mismatched elsif !@repo.git_dir.exist? :mismatched elsif !@repo.git_dir.directory? :mismatched elsif !(@repo.origin == @remote) :mismatched - elsif @repo.dirty? + elsif @repo.head.nil? + :mismatched + elsif @repo.dirty?(exclude_spec) :dirty elsif !(@repo.head == @cache.resolve(ref)) :outdated @@ -93,6 +102,7 @@ # @api private def sync_cache?(ref) return true if !@cache.exist? + return true if ref == 'HEAD' return true if !([:commit, :tag].include? @cache.ref_type(ref)) return false end diff -Nru r10k-3.7.0/lib/r10k/git.rb r10k-4.0.0/lib/r10k/git.rb --- r10k-3.7.0/lib/r10k/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/git.rb 2023-08-01 15:06:51.000000000 +0000 @@ -134,6 +134,10 @@ extend R10K::Settings::Mixin::ClassMethods def_setting_attr :private_key + def_setting_attr :oauth_token + def_setting_attr :github_app_id + def_setting_attr :github_app_key + def_setting_attr :github_app_ttl def_setting_attr :proxy def_setting_attr :username def_setting_attr :repositories, {} diff -Nru r10k-3.7.0/lib/r10k/initializers.rb r10k-4.0.0/lib/r10k/initializers.rb --- r10k-3.7.0/lib/r10k/initializers.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/initializers.rb 2023-08-01 15:06:51.000000000 +0000 @@ -4,6 +4,7 @@ require 'r10k/git/cache' require 'r10k/forge/module_release' +require 'r10k/tarball' module R10K module Initializers @@ -30,14 +31,27 @@ logger.warn(_("the purgedirs key in r10k.yaml is deprecated. it is currently ignored.")) end + with_setting(:logging) { |value| LoggingInitializer.new(value).call } + with_setting(:deploy) { |value| DeployInitializer.new(value).call } with_setting(:cachedir) { |value| R10K::Git::Cache.settings[:cache_root] = value } with_setting(:cachedir) { |value| R10K::Forge::ModuleRelease.settings[:cache_root] = value } + with_setting(:cachedir) { |value| R10K::Tarball.settings[:cache_root] = value } with_setting(:pool_size) { |value| R10K::Puppetfile.settings[:pool_size] = value } + with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value } with_setting(:git) { |value| GitInitializer.new(value).call } with_setting(:forge) { |value| ForgeInitializer.new(value).call } + with_setting(:tarball) { |value| TarballInitializer.new(value).call } + end + end + + class LoggingInitializer < BaseInitializer + def call + with_setting(:level) { |value| R10K::Logging.level = value } + with_setting(:disable_default_stderr) { |value| R10K::Logging.disable_default_stderr = value } + with_setting(:outputs) { |value| R10K::Logging.add_outputters(value) } end end @@ -55,6 +69,10 @@ with_setting(:private_key) { |value| R10K::Git.settings[:private_key] = value } with_setting(:proxy) { |value| R10K::Git.settings[:proxy] = value } with_setting(:repositories) { |value| R10K::Git.settings[:repositories] = value } + with_setting(:oauth_token) { |value| R10K::Git.settings[:oauth_token] = value } + with_setting(:github_app_id) { |value| R10K::Git.settings[:github_app_id] = value } + with_setting(:github_app_key) { |value| R10K::Git.settings[:github_app_key] = value } + with_setting(:github_app_ttl) { |value| R10K::Git.settings[:github_app_ttl] = value } end end @@ -62,6 +80,13 @@ def call with_setting(:baseurl) { |value| PuppetForge.host = value } with_setting(:proxy) { |value| PuppetForge::Connection.proxy = value } + with_setting(:authorization_token) { |value| PuppetForge::Connection.authorization = value } + end + end + + class TarballInitializer < BaseInitializer + def call + with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value } end end end diff -Nru r10k-3.7.0/lib/r10k/logging.rb r10k-4.0.0/lib/r10k/logging.rb --- r10k-3.7.0/lib/r10k/logging.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/logging.rb 2023-08-01 15:06:51.000000000 +0000 @@ -8,6 +8,16 @@ module R10K::Logging LOG_LEVELS = %w{DEBUG2 DEBUG1 DEBUG INFO NOTICE WARN ERROR FATAL} + SYSLOG_LEVELS_MAP = { + 'DEBUG2' => 'DEBUG', + 'DEBUG1' => 'DEBUG', + 'DEBUG' => 'DEBUG', + 'INFO' => 'INFO', + 'NOTICE' => 'INFO', + 'WARN' => 'WARN', + 'ERROR' => 'ERROR', + 'FATAL' => 'FATAL', + }.freeze def logger_name self.class.to_s @@ -21,6 +31,9 @@ else @logger = Log4r::Logger.new(name) @logger.add(R10K::Logging.outputter) + R10K::Logging.outputters.each do |output| + @logger.add(output) + end end end @logger @@ -59,7 +72,7 @@ if level.nil? raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % {val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect} end - outputter.level = level + outputter.level = level unless @disable_default_stderr @level = level if level < Log4r::INFO @@ -69,6 +82,58 @@ end end + def disable_default_stderr=(val) + @disable_default_stderr = val + outputter.level = val ? Log4r::OFF : @level + end + + def add_outputters(outputs) + outputs.each do |output| + type = output.fetch(:type) + # Support specifying both short as well as full names + type = type.to_s[0..-10] if type.to_s.downcase.end_with? 'outputter' + + name = output.fetch(:name, 'r10k') + if output[:level] + level = parse_level(output[:level]) + if level.nil? + raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: output[:level], log_levels: LOG_LEVELS.map(&:downcase).inspect } + end + else + level = self.level + end + only_at = output[:only_at] + only_at&.map! do |val| + lv = parse_level(val) + if lv.nil? + raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect } + end + + lv + end + parameters = output.fetch(:parameters, {}).merge({ level: level }) + + begin + # Try to load the outputter file if possible + require "log4r/outputter/#{type.to_s.downcase}outputter" + rescue LoadError + false + end + outputtertype = Log4r.constants + .select { |klass| klass.to_s.end_with? 'Outputter' } + .find { |klass| klass.to_s.downcase == "#{type.to_s.downcase}outputter" } + raise ArgumentError, "Unable to find a #{output[:type]} outputter." unless outputtertype + + outputter = Log4r.const_get(outputtertype).new(name, parameters) + outputter.only_at(*only_at) if only_at + # Handle log4r's syslog mapping correctly + outputter.map_levels_by_name_to_syslog(SYSLOG_LEVELS_MAP) if outputter.respond_to? :map_levels_by_name_to_syslog + + @outputters << outputter + Log4r::Logger.global.add outputter + end + end + extend Forwardable def_delegators :@outputter, :use_color, :use_color= @@ -87,6 +152,16 @@ # @return [Log4r::Outputter] attr_reader :outputter + # @!attribute [r] outputters + # @api private + # @return [Array[Log4r::Outputter]] + attr_reader :outputters + + # @!attribute [r] disable_default_stderr + # @api private + # @return [Boolean] + attr_reader :disable_default_stderr + def default_formatter Log4r::PatternFormatter.new(:pattern => '%l\t -> %m') end @@ -106,4 +181,6 @@ @level = Log4r::WARN @formatter = default_formatter @outputter = default_outputter + @outputters = [] + @disable_default_stderr = false end diff -Nru r10k-3.7.0/lib/r10k/module/base.rb r10k-4.0.0/lib/r10k/module/base.rb --- r10k-3.7.0/lib/r10k/module/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module/base.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,9 +1,12 @@ require 'r10k/module' +require 'r10k/logging' require 'puppet_forge' # This class defines a common interface for module implementations. class R10K::Module::Base + include R10K::Logging + # @!attribute [r] title # @return [String] The forward slash separated owner and name of the module attr_reader :title @@ -35,6 +38,10 @@ # @return [String] Where the module was sourced from. E.g., "Puppetfile" attr_accessor :origin + # @!attribute [rw] spec_deletable + # @return [Boolean] set this to true if the spec dir can be safely removed, ie in the moduledir + attr_accessor :spec_deletable + # There's been some churn over `author` vs `owner` and `full_name` over # `title`, so in the short run it's easier to support both and deprecate one # later. @@ -43,7 +50,7 @@ # @param title [String] # @param dirname [String] - # @param args [Array] + # @param args [Hash] def initialize(title, dirname, args, environment=nil) @title = PuppetForge::V3.normalize_name(title) @dirname = dirname @@ -51,7 +58,18 @@ @owner, @name = parse_title(@title) @path = Pathname.new(File.join(@dirname, @name)) @environment = environment + @overrides = args.delete(:overrides) || {} + @spec_deletable = true + @exclude_spec = true + @exclude_spec = @overrides.dig(:modules, :exclude_spec) unless @overrides.dig(:modules, :exclude_spec).nil? + if args.has_key?(:exclude_spec) + logger.debug2 _("Overriding :exclude_spec setting with per module setting for #{@title}") + @exclude_spec = args.delete(:exclude_spec) + end @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value + + @requested_modules = @overrides.dig(:modules, :requested_modules) || [] + @should_sync = (@requested_modules.empty? || @requested_modules.include?(@name)) end # @deprecated @@ -60,12 +78,54 @@ path.to_s end + # Delete the spec dir if @exclude_spec is true and @spec_deletable is also true + def maybe_delete_spec_dir + if @exclude_spec + if @spec_deletable + delete_spec_dir + else + logger.info _("Spec dir for #{@title} will not be deleted because it is not in the moduledir") + end + end + end + + # Actually remove the spec dir + def delete_spec_dir + spec_path = @path + 'spec' + if spec_path.symlink? + spec_path = spec_path.realpath + end + if spec_path.directory? + logger.debug2 _("Deleting spec data at #{spec_path}") + # Use the secure flag for the #rm_rf method to avoid security issues + # involving TOCTTOU(time of check to time of use); more details here: + # https://ruby-doc.org/stdlib-2.7.0/libdoc/fileutils/rdoc/FileUtils.html#method-c-rm_rf + # Additionally, #rm_rf also has problems in windows with with symlink targets + # also being deleted; this should be revisted if Windows becomes higher priority. + FileUtils.rm_rf(spec_path, secure: true) + else + logger.debug2 _("No spec dir detected at #{spec_path}, skipping deletion") + end + end + # Synchronize this module with the indicated state. - # @abstract + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) raise NotImplementedError end + def should_sync? + if @should_sync + logger.info _("Deploying module to %{path}") % {path: path} + true + else + logger.debug1(_("Only updating modules %{modules}, skipping module %{name}") % {modules: @requested_modules.inspect, name: name}) + false + end + end + + # Return the desired version of this module # @abstract def version @@ -87,6 +147,7 @@ raise NotImplementedError end + # Deprecated def accept(visitor) visitor.visit(:module, self) end diff -Nru r10k-3.7.0/lib/r10k/module/definition.rb r10k-4.0.0/lib/r10k/module/definition.rb --- r10k-3.7.0/lib/r10k/module/definition.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module/definition.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,64 @@ +require 'r10k/module' + +class R10K::Module::Definition < R10K::Module::Base + + attr_reader :version + + def initialize(name, dirname:, args:, implementation:, environment: nil) + @original_name = name + @original_args = args.dup + @implementation = implementation + @version = implementation.statically_defined_version(name, args) + + super(name, dirname, args, environment) + end + + def to_implementation + mod = @implementation.new(@title, @dirname, @original_args, @environment) + + mod.origin = origin + mod.spec_deletable = spec_deletable + + mod + end + + # syncing is a noop for module definitions + # Returns false to inidicate the module was not updated + def sync(args = {}) + logger.debug1(_("Not updating module %{name}, assuming content unchanged") % {name: name}) + false + end + + def status + :insync + end + + def properties + type = nil + + if @args[:type] + type = @args[:type] + elsif @args[:ref] || @args[:commit] || @args[:branch] || @args[:tag] + type = 'git' + elsif @args[:svn] + # This logic is clear and included for completeness sake, though at + # this time module definitions do not support SVN versions. + type = 'svn' + else + type = 'forge' + end + + { + expected: version, + # We can't get the value for `actual` here because that requires the + # implementation (and potentially expensive operations by the + # implementation). Some consumers will check this value, if it exists + # and if not, fall back to the expected version. That is the correct + # behavior when assuming modules are unchanged, and why `actual` is set + # to `nil` here. + actual: nil, + type: type + } + end +end + diff -Nru r10k-3.7.0/lib/r10k/module/forge.rb r10k-4.0.0/lib/r10k/module/forge.rb --- r10k-3.7.0/lib/r10k/module/forge.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module/forge.rb 2023-08-01 15:06:51.000000000 +0000 @@ -13,11 +13,11 @@ R10K::Module.register(self) def self.implement?(name, args) - !!(name.match %r[\w+[/-]\w+]) && valid_version?(args) + args[:type].to_s == 'forge' end - def self.valid_version?(expected_version) - expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version) + def self.statically_defined_version(name, args) + args[:version] if args[:version].is_a?(String) end # @!attribute [r] metadata @@ -30,27 +30,54 @@ # @return [PuppetForge::V3::Module] The Puppet Forge module metadata attr_reader :v3_module - include R10K::Logging + include R10K::Util::Setopts - def initialize(title, dirname, expected_version, environment=nil) + def initialize(title, dirname, opts, environment=nil) super @metadata_file = R10K::Module::MetadataFile.new(path + 'metadata.json') @metadata = @metadata_file.read - @expected_version = expected_version || current_version || :latest + setopts(opts, { + # Standard option interface + :version => :expected_version, + :source => ::R10K::Util::Setopts::Ignore, + :type => ::R10K::Util::Setopts::Ignore, + }, :raise_on_unhandled => false) + + # Validate version and raise on issue. Title is validated by base class. + unless valid_version?(@expected_version) + raise ArgumentError, _("Module version %{ver} is not a valid Forge module version") % {ver: @expected_version} + end + + @expected_version ||= current_version || :latest + @v3_module = PuppetForge::V3::Module.new(:slug => @title) end + def valid_version?(version) + version == :latest || version.nil? || PuppetForge::Util.version_valid?(version) + end + + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) - case status - when :absent - install - when :outdated - upgrade - when :mismatched - reinstall + updated = false + if should_sync? + case status + when :absent + install + updated = true + when :outdated + upgrade + updated = true + when :mismatched + reinstall + updated = true + end + maybe_delete_spec_dir end + updated end def properties @@ -65,7 +92,11 @@ def expected_version if @expected_version == :latest begin - @expected_version = @v3_module.current_release.version + if @v3_module.current_release + @expected_version = @v3_module.current_release.version + else + raise PuppetForge::ReleaseNotFound, _("The module %{title} does not appear to have any published releases, cannot determine latest version.") % { title: @title } + end rescue Faraday::ResourceNotFound => e raise PuppetForge::ReleaseNotFound, _("The module %{title} does not exist on %{url}.") % {title: @title, url: PuppetForge::V3::Release.conn.url_prefix}, e.backtrace end @@ -171,7 +202,7 @@ if (match = title.match(/\A(\w+)[-\/](\w+)\Z/)) [match[1], match[2]] else - raise ArgumentError, _("Forge module names must match 'owner/modulename'") + raise ArgumentError, _("Forge module names must match 'owner/modulename', instead got #{title}") end end end diff -Nru r10k-3.7.0/lib/r10k/module/git.rb r10k-4.0.0/lib/r10k/module/git.rb --- r10k-3.7.0/lib/r10k/module/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module/git.rb 2023-08-01 15:06:51.000000000 +0000 @@ -8,9 +8,22 @@ R10K::Module.register(self) def self.implement?(name, args) - args.is_a? Hash and args.has_key?(:git) - rescue - false + args.has_key?(:git) || args[:type].to_s == 'git' + end + + # Will be called if self.implement? above returns true. Will return + # the version info, if version is statically defined in the modules + # declaration. + def self.statically_defined_version(name, args) + if !args[:type] && (args[:ref] || args[:tag] || args[:commit]) + if args[:ref] && args[:ref].to_s.match(/[0-9a-f]{40}/) + args[:ref] + else + args[:tag] || args[:commit] + end + elsif args[:type].to_s == 'git' && args[:version] && args[:version].to_s.match(/[0-9a-f]{40}/) + args[:version] + end end # @!attribute [r] repo @@ -28,16 +41,49 @@ # @return [String] attr_reader :default_ref - def initialize(title, dirname, args, environment=nil) - super + # @!attribute [r] default_override_ref + # @api private + # @return [String] + attr_reader :default_override_ref + + include R10K::Util::Setopts - parse_options(@args) + def initialize(title, dirname, opts, environment=nil) + + super + setopts(opts, { + # Standard option interface + :version => :desired_ref, + :source => :remote, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :branch => :desired_ref, + :tag => :desired_ref, + :commit => :desired_ref, + :ref => :desired_ref, + :git => :remote, + :default_branch => :default_branch, + :default_branch_override => :default_override_ref, + }, :raise_on_unhandled => false) + + @default_ref = @default_branch.nil? ? @overrides.dig(:modules, :default_ref) : @default_branch + force = @overrides[:force] + @force = force == false ? false : true + + if @desired_ref == :control_branch + if @environment && @environment.respond_to?(:ref) + @desired_ref = @environment.ref + else + logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available." % {name: name}) + end + end @repo = R10K::Git::StatefulRepository.new(@remote, @dirname, @name) end def version - validate_ref(@desired_ref, @default_ref) + validate_ref(@desired_ref, @default_ref, @default_override_ref) end def properties @@ -48,9 +94,17 @@ } end + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) - force = opts && opts.fetch(:force, true) - @repo.sync(version, force) + force = opts[:force] || @force + if should_sync? + updated = @repo.sync(version, force, @exclude_spec) + else + updated = false + end + maybe_delete_spec_dir + updated end def status @@ -61,11 +115,21 @@ @repo.cache.sanitized_dirname end + def validate_ref_defined + if @desired_ref.nil? && @default_ref.nil? && @default_override_ref.nil? + msg = "No ref defined for module #{@name}. Add a ref to the module definition " + msg << "or set git:default_ref in the r10k.yaml config to configure a global default ref." + raise ArgumentError, msg + end + end + private - def validate_ref(desired, default) + def validate_ref(desired, default, default_override) if desired && desired != :control_branch && @repo.resolve(desired) return desired + elsif default_override && @repo.resolve(default_override) + return default_override elsif default && @repo.resolve(default) return default else @@ -81,33 +145,21 @@ msg << "Could not determine desired ref" end + if default_override + msg << "or resolve the default branch override '%{default_override}'," + vars[:default_override] = default_override + end + if default msg << "or resolve default ref '%{default}'" vars[:default] = default else - msg << "and no default provided" + msg << "and no default provided. r10k no longer hardcodes 'master' as the default ref." + msg << "Consider setting a ref per module in the Puppetfile or setting git:default_ref" + msg << "in your r10k config." end raise ArgumentError, _(msg.join(' ')) % vars end end - - def parse_options(options) - ref_opts = [:branch, :tag, :commit, :ref] - known_opts = [:git, :default_branch] + ref_opts - - unhandled = options.keys - known_opts - unless unhandled.empty? - raise ArgumentError, _("Unhandled options %{unhandled} specified for %{class}") % {unhandled: unhandled, class: self.class} - end - - @remote = options[:git] - - @desired_ref = ref_opts.find { |key| break options[key] if options.has_key?(key) } || 'master' - @default_ref = options[:default_branch] - - if @desired_ref == :control_branch && @environment && @environment.respond_to?(:ref) - @desired_ref = @environment.ref - end - end end diff -Nru r10k-3.7.0/lib/r10k/module/local.rb r10k-4.0.0/lib/r10k/module/local.rb --- r10k-3.7.0/lib/r10k/module/local.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module/local.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,5 +1,4 @@ require 'r10k/module' -require 'r10k/logging' # A dummy module type that can be used to "protect" Puppet modules that exist # inside of the Puppetfile "moduledir" location. Local modules will not be @@ -9,13 +8,15 @@ R10K::Module.register(self) def self.implement?(name, args) - args.is_a?(Hash) && args[:local] + args[:local] || args[:type].to_s == 'local' end - include R10K::Logging + def self.statically_defined_version(*) + "0.0.0" + end def version - "0.0.0" + self.class.statically_defined_version end def properties @@ -30,7 +31,10 @@ :insync end + # @param [Hash] opts Deprecated + # @return [Boolean] false, because local modules are always considered in-sync def sync(opts={}) logger.debug1 _("Module %{title} is a local module, always indicating synced.") % {title: title} + false end end diff -Nru r10k-3.7.0/lib/r10k/module/svn.rb r10k-4.0.0/lib/r10k/module/svn.rb --- r10k-3.7.0/lib/r10k/module/svn.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module/svn.rb 2023-08-01 15:06:51.000000000 +0000 @@ -7,7 +7,11 @@ R10K::Module.register(self) def self.implement?(name, args) - args.is_a? Hash and args.has_key? :svn + args.has_key?(:svn) || args[:type].to_s == 'svn' + end + + def self.statically_defined_version(name, args) + nil end # @!attribute [r] expected_revision @@ -36,18 +40,21 @@ include R10K::Util::Setopts - INITIALIZE_OPTS = { - :svn => :url, - :rev => :expected_revision, - :revision => :expected_revision, - :username => :self, - :password => :self - } - def initialize(name, dirname, opts, environment=nil) super - - setopts(opts, INITIALIZE_OPTS) + setopts(opts, { + # Standard option interface + :source => :url, + :version => :expected_revision, + :type => ::R10K::Util::Setopts::Ignore, + + # Type-specific options + :svn => :url, + :rev => :expected_revision, + :revision => :expected_revision, + :username => :self, + :password => :self + }, :raise_on_unhandled => false) @working_dir = R10K::SVN::WorkingDir.new(@path, :username => @username, :password => @password) end @@ -66,15 +73,25 @@ end end + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise def sync(opts={}) - case status - when :absent - install - when :mismatched - reinstall - when :outdated - update + updated = false + if should_sync? + case status + when :absent + install + updated = true + when :mismatched + reinstall + updated = true + when :outdated + update + updated = true + end + maybe_delete_spec_dir end + updated end def exist? diff -Nru r10k-3.7.0/lib/r10k/module/tarball.rb r10k-4.0.0/lib/r10k/module/tarball.rb --- r10k-3.7.0/lib/r10k/module/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module/tarball.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,101 @@ +require 'r10k/module' +require 'r10k/util/setopts' +require 'r10k/tarball' + +# This class defines a tarball source module implementation +class R10K::Module::Tarball < R10K::Module::Base + + R10K::Module.register(self) + + def self.implement?(name, args) + args.is_a?(Hash) && args[:type].to_s == 'tarball' + rescue + false + end + + def self.statically_defined_version(name, args) + args[:version] || args[:checksum] + end + + # @!attribute [r] tarball + # @api private + # @return [R10K::Tarball] + attr_reader :tarball + + include R10K::Util::Setopts + + def initialize(name, dirname, opts, environment=nil) + super + setopts(opts, { + # Standard option interface + :source => :self, + :version => :checksum, + :type => ::R10K::Util::Setopts::Ignore, + :overrides => :self, + + # Type-specific options + :checksum => :self, + }) + + @tarball = R10K::Tarball.new(name, @source, checksum: @checksum) + end + + # Return the status of the currently installed module. + # + # @return [Symbol] + def status + if not path.exist? + :absent + elsif not (tarball.cache_valid? && tarball.insync?(path.to_s)) + :mismatched + else + :insync + end + end + + # Synchronize this module with the indicated state. + # @param [Hash] opts Deprecated + # @return [Boolean] true if the module was updated, false otherwise + def sync(opts={}) + tarball.get unless tarball.cache_valid? + if should_sync? + case status + when :absent + tarball.unpack(path.to_s) + when :mismatched + path.rmtree + tarball.unpack(path.to_s) + end + maybe_delete_spec_dir + true + else + false + end + end + + # Return the desired version of this module + def version + @checksum || '(present)' + end + + # Return the properties of the module + # + # @return [Hash] + # @abstract + def properties + { + :expected => version, + :actual => ((state = status) == :insync) ? version : state, + :type => :tarball, + } + end + + # Tarball caches are files, not directories. An important purpose of this + # method is to indicate where the cache "path" is, for locking/parallelism, + # so for the Tarball module type, the relevant path location is returned. + # + # @return [String] The path this module will cache its tarball source to + def cachedir + tarball.cache_path + end +end diff -Nru r10k-3.7.0/lib/r10k/module.rb r10k-4.0.0/lib/r10k/module.rb --- r10k-3.7.0/lib/r10k/module.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module.rb 2023-08-01 15:06:51.000000000 +0000 @@ -17,22 +17,41 @@ # # @param [String] name The unique name of the module # @param [String] basedir The root to install the module in - # @param [Object] args An arbitary value or set of values that specifies the implementation + # @param [Hash] args An arbitary Hash that specifies the implementation # @param [R10K::Environment] environment Optional environment that this module is a part of # # @return [Object < R10K::Module] A member of the implementing subclass def self.new(name, basedir, args, environment=nil) + with_implementation(name, args) do |implementation| + implementation.new(name, basedir, args, environment) + end + end + + # Takes the same signature as Module.new but returns an metadata module + def self.from_metadata(name, basedir, args, environment=nil) + with_implementation(name, args) do |implementation| + R10K::Module::Definition.new(name, + dirname: basedir, + args: args, + implementation: implementation, + environment: environment) + end + end + + def self.with_implementation(name, args, &block) if implementation = @klasses.find { |klass| klass.implement?(name, args) } - obj = implementation.new(name, basedir, args, environment) - obj + block.call(implementation) else raise _("Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)") % {name: name, args: args.inspect} end end + require 'r10k/module/base' require 'r10k/module/git' require 'r10k/module/svn' require 'r10k/module/local' require 'r10k/module/forge' + require 'r10k/module/definition' + require 'r10k/module/tarball' end diff -Nru r10k-3.7.0/lib/r10k/module_loader/puppetfile/dsl.rb r10k-4.0.0/lib/r10k/module_loader/puppetfile/dsl.rb --- r10k-3.7.0/lib/r10k/module_loader/puppetfile/dsl.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module_loader/puppetfile/dsl.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,42 @@ +module R10K + module ModuleLoader + class Puppetfile + class DSL + # A barebones implementation of the Puppetfile DSL + # + # @api private + + def initialize(librarian, metadata_only: false) + @librarian = librarian + @metadata_only = metadata_only + end + + def mod(name, args = nil) + if args.is_a?(Hash) + opts = args + else + opts = { type: 'forge', version: args } + end + + if @metadata_only + @librarian.add_module_metadata(name, opts) + else + @librarian.add_module(name, opts) + end + end + + def forge(location) + @librarian.set_forge(location) + end + + def moduledir(location) + @librarian.set_moduledir(location) + end + + def method_missing(method, *args) + raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method} + end + end + end + end +end diff -Nru r10k-3.7.0/lib/r10k/module_loader/puppetfile.rb r10k-4.0.0/lib/r10k/module_loader/puppetfile.rb --- r10k-3.7.0/lib/r10k/module_loader/puppetfile.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/module_loader/puppetfile.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,292 @@ +require 'r10k/errors' +require 'r10k/logging' +require 'r10k/module' +require 'r10k/module_loader/puppetfile/dsl' + +require 'pathname' + +module R10K + module ModuleLoader + class Puppetfile + + include R10K::Logging + + DEFAULT_MODULEDIR = 'modules' + DEFAULT_PUPPETFILE_NAME = 'Puppetfile' + + attr_accessor :default_branch_override, :environment + attr_reader :modules, :moduledir, :puppetfile_path, + :managed_directories, :desired_contents, :purge_exclusions, + :environment_name + + # @param basedir [String] The path that contains the moduledir & + # Puppetfile by default. May be an environment, project, or + # simple directory. + # @param puppetfile [String] The path to the Puppetfile, either an + # absolute full path or a relative path with regards to the basedir. + # @param moduledir [String] The path to the moduledir, either an + # absolute full path or a relative path with regards to the basedir. + # @param forge [String] The url (without protocol) to the Forge + # @param overrides [Hash] Configuration for loaded modules' behavior + # @param environment [R10K::Environment] When provided, the environment + # in which loading takes place + # @param module_exclude_regex [Regex] A regex to exclude modules from + # installation. Helpful in CI environments. + def initialize(basedir:, + moduledir: DEFAULT_MODULEDIR, + puppetfile: DEFAULT_PUPPETFILE_NAME, + overrides: {}, + environment: nil, + module_exclude_regex: nil) + + @basedir = cleanpath(basedir) + @moduledir = resolve_path(@basedir, moduledir) + @puppetfile_path = resolve_path(@basedir, puppetfile) + @overrides = overrides + @environment = environment + @module_exclude_regex = module_exclude_regex + @environment_name = @environment&.name + @default_branch_override = @overrides.dig(:environments, :default_branch_override) + @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override) + + @existing_module_metadata = [] + @existing_module_versions_by_name = {} + @modules = [] + + @managed_directories = [] + @desired_contents = [] + @purge_exclusions = [] + end + + def load + with_readable_puppetfile(@puppetfile_path) do + self.load! + end + end + + def load! + logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path} + logger.debug _("Using moduledir '%{moduledir}'") % {moduledir: @moduledir} + + dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self) + dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path) + + validate_no_duplicate_names(@modules) + @modules = filter_modules(@modules, @module_exclude_regex) if @module_exclude_regex + + managed_content = @modules.group_by(&:dirname) + + @managed_directories = determine_managed_directories(managed_content) + @desired_contents = determine_desired_contents(managed_content) + @purge_exclusions = determine_purge_exclusions(@managed_directories) + + { + modules: @modules, + managed_directories: @managed_directories, + desired_contents: @desired_contents, + purge_exclusions: @purge_exclusions + } + + rescue SyntaxError, LoadError, ArgumentError, NameError => e + raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path}) + end + + def load_metadata + with_readable_puppetfile(@puppetfile_path) do + self.load_metadata! + end + end + + def load_metadata! + dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self, metadata_only: true) + dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path) + + @existing_module_versions_by_name = @existing_module_metadata.map {|mod| [ mod.name, mod.version ] }.to_h + empty_load_output.merge(modules: @existing_module_metadata) + + rescue SyntaxError, LoadError, ArgumentError, NameError => e + logger.warn _("Unable to preload Puppetfile because of %{msg}" % { msg: e.message }) + end + + def add_module_metadata(name, info) + install_path, metadata_info, _ = parse_module_definition(name, info) + + mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment) + + @existing_module_metadata << mod + end + + ## + ## set_forge, set_moduledir, and add_module are used directly by the DSL class + ## + + # @param [String] forge + def set_forge(forge) + if @allow_puppetfile_forge + logger.debug _("Using Forge from Puppetfile: %{forge}") % { forge: forge } + PuppetForge.host = forge + else + logger.debug _("Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}.") % { forge: PuppetForge.host } + end + end + + # @param [String] moduledir + def set_moduledir(moduledir) + @moduledir = resolve_path(@basedir, moduledir) + end + + # @param [String] name + # @param [Hash, String, Symbol, nil] info Calling with + # anything but a Hash is deprecated. The DSL will now convert + # String and Symbol versions to Hashes of the shape + # { version: } + # + # String inputs should be valid module versions, the Symbol + # `:latest` is allowed, as well as `nil`. + # + # Non-Hash inputs are only ever used by Forge modules. In + # future versions this method will require the caller (the + # DSL class, not the Puppetfile author) to do this conversion + # itself. + # + def add_module(name, info) + install_path, metadata_info, spec_deletable = parse_module_definition(name, info) + + mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment) + mod.origin = :puppetfile + mod.spec_deletable = spec_deletable + + # Do not save modules if they would conflict with the attached + # environment + if @environment && @environment.module_conflicts?(mod) + return @modules + end + + # If this module's metadata has a static version, and that version + # matches the existing module declaration, and it ostensibly + # has already has been deployed to disk, use it. Otherwise create a + # regular module to sync. + unless mod.version && + mod.version == @existing_module_versions_by_name[mod.name] && + File.directory?(mod.path) + mod = mod.to_implementation + end + + @modules << mod + end + + private + + def empty_load_output + { + modules: [], + managed_directories: [], + desired_contents: [], + purge_exclusions: [] + } + end + + def with_readable_puppetfile(puppetfile_path, &block) + if File.readable?(puppetfile_path) + block.call + else + logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect} + + empty_load_output + end + end + + def parse_module_definition(name, info) + # The only valid (deprecated) way a module can be defined with a + # non-hash info is if it is a Forge module. + if !info.is_a?(Hash) + info = { type: 'forge', version: info } + end + + info[:overrides] = @overrides + + if @default_branch_override + info[:default_branch_override] = @default_branch_override + end + + spec_deletable = false + if install_path = info.delete(:install_path) + install_path = resolve_path(@basedir, install_path) + validate_install_path(install_path, name) + else + install_path = @moduledir + spec_deletable = true + end + + return [ install_path, info, spec_deletable ] + end + + def filter_modules(modules, exclude_regex) + modules.reject { |mod| mod.name =~ /#{exclude_regex}/ } + end + + # @param [Array] modules + def validate_no_duplicate_names(modules) + dupes = modules + .group_by { |mod| mod.name } + .select { |_, mods| mods.size > 1 } + .map(&:first) + unless dupes.empty? + msg = _('Puppetfiles cannot contain duplicate module names.') + msg += ' ' + msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') }) + raise R10K::Error.new(msg) + end + end + + def resolve_path(base, path) + if Pathname.new(path).absolute? + cleanpath(path) + else + cleanpath(File.join(base, path)) + end + end + + def validate_install_path(path, modname) + unless /^#{Regexp.escape(@basedir)}.*/ =~ path + raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{@basedir}") + end + + true + end + + def determine_managed_directories(managed_content) + managed_content.keys.reject { |dir| dir == @basedir } + end + + # Returns an array of the full paths to all the content being managed. + # @return [Array] + def determine_desired_contents(managed_content) + managed_content.flat_map do |install_path, mods| + mods.collect { |mod| File.join(install_path, mod.name) } + end + end + + def determine_purge_exclusions(managed_dirs) + if environment && environment.respond_to?(:desired_contents) + managed_dirs + environment.desired_contents + else + managed_dirs + end + end + + # .cleanpath is as close to a canonical path as we can do without touching + # the filesystem. The .realpath methods will choke if some of the + # intermediate paths are missing, even though in some cases we will create + # them later as needed. + def cleanpath(path) + Pathname.new(path).cleanpath.to_s + end + + # For testing purposes only + def puppetfile_content(path) + File.read(path) + end + end + end +end diff -Nru r10k-3.7.0/lib/r10k/puppetfile.rb r10k-4.0.0/lib/r10k/puppetfile.rb --- r10k-3.7.0/lib/r10k/puppetfile.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/puppetfile.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,10 +3,18 @@ require 'r10k/module' require 'r10k/util/purgeable' require 'r10k/errors' +require 'r10k/content_synchronizer' +require 'r10k/module_loader/puppetfile/dsl' +require 'r10k/module_loader/puppetfile' module R10K + +# Deprecated, use R10K::ModuleLoader::Puppetfile#load to load content, +# provide the `:modules` key of the returned Hash to +# R10K::ContentSynchronizer (either the `serial_sync` or `concurrent_sync`) +# and the remaining keys (`:managed_directories`, `:desired_contents`, and +# `:purge_exclusions`) to R10K::Util::Cleaner. class Puppetfile - # Defines the data members of a Puppetfile include R10K::Settings::Mixin @@ -18,289 +26,186 @@ # @return [String] The URL to use for the Puppet Forge attr_reader :forge - # @!attribute [r] modules - # @return [Array] - attr_reader :modules - # @!attribute [r] basedir # @return [String] The base directory that contains the Puppetfile attr_reader :basedir - # @!attribute [r] moduledir - # @return [String] The directory to install the modules #{basedir}/modules - attr_reader :moduledir - - # @!attrbute [r] puppetfile_path - # @return [String] The path to the Puppetfile - attr_reader :puppetfile_path - - # @!attribute [rw] environment + # @!attribute [r] environment # @return [R10K::Environment] Optional R10K::Environment that this Puppetfile belongs to. - attr_accessor :environment + attr_reader :environment # @!attribute [rw] force # @return [Boolean] Overwrite any locally made changes attr_accessor :force - # @!attribute [r] modules_by_vcs_cachedir - # @api private Only exposed for testing purposes - # @return [Hash{:none, String => Array}] - attr_reader :modules_by_vcs_cachedir + # @!attribute [r] overrides + # @return [Hash] Various settings overridden from normal configs + attr_reader :overrides + + # @!attribute [r] loader + # @return [R10K::ModuleLoader::Puppetfile] The internal module loader + attr_reader :loader # @param [String] basedir - # @param [String] moduledir The directory to install the modules, default to #{basedir}/modules - # @param [String] puppetfile_path The path to the Puppetfile, default to #{basedir}/Puppetfile - # @param [String] puppetfile_name The name of the Puppetfile, default to 'Puppetfile' - # @param [Boolean] force Shall we overwrite locally made changes? - def initialize(basedir, moduledir = nil, puppetfile_path = nil, puppetfile_name = nil, force = nil ) + # @param [Hash, String, nil] options_or_moduledir The directory to install the modules or a Hash of options. + # Usage as moduledir is deprecated. Only use as options, defaults to nil + # @param [String, nil] puppetfile_path Deprecated - The path to the Puppetfile, defaults to nil + # @param [String, nil] puppetfile_name Deprecated - The name of the Puppetfile, defaults to nil + # @param [Boolean, nil] force Deprecated - Shall we overwrite locally made changes? + def initialize(basedir, options_or_moduledir = nil, deprecated_path_arg = nil, deprecated_name_arg = nil, deprecated_force_arg = nil) @basedir = basedir - @force = force || false - @moduledir = moduledir || File.join(basedir, 'modules') - @puppetfile_name = puppetfile_name || 'Puppetfile' - @puppetfile_path = puppetfile_path || File.join(basedir, @puppetfile_name) - - logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path} - - @modules = [] - @managed_content = {} - @modules_by_vcs_cachedir = {} - @forge = 'forgeapi.puppetlabs.com' + if options_or_moduledir.is_a? Hash + options = options_or_moduledir + deprecated_moduledir_arg = nil + else + options = {} + deprecated_moduledir_arg = options_or_moduledir + end + + @force = deprecated_force_arg || options.delete(:force) || false + @moduledir = deprecated_moduledir_arg || options.delete(:moduledir) || File.join(basedir, 'modules') + puppetfile_name = deprecated_name_arg || options.delete(:puppetfile_name) || 'Puppetfile' + puppetfile_path = deprecated_path_arg || options.delete(:puppetfile_path) + @puppetfile = puppetfile_path || puppetfile_name + @environment = options.delete(:environment) + + @overrides = options.delete(:overrides) || {} + @default_branch_override = @overrides.dig(:environments, :default_branch_override) + + @forge = 'forgeapi.puppet.com' + + @loader = ::R10K::ModuleLoader::Puppetfile.new( + basedir: @basedir, + moduledir: @moduledir, + puppetfile: @puppetfile, + overrides: @overrides, + environment: @environment + ) + + @loaded_content = { + modules: [], + managed_directories: [], + desired_contents: [], + purge_exclusions: [] + } @loaded = false end + # @param [String] default_branch_override The default branch to use + # instead of one specified in the module declaration, if applicable. + # Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass + # the default_branch_override as an option on initialization. def load(default_branch_override = nil) - return true if self.loaded? - if File.readable? @puppetfile_path - self.load!(default_branch_override) + if self.loaded? + return @loaded_content else - logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect} + if !File.readable?(puppetfile_path) + logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect} + else + self.load!(default_branch_override) + end end end + # @param [String] default_branch_override The default branch to use + # instead of one specified in the module declaration, if applicable. + # Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass + # the default_branch_override as an option on initialization. def load!(default_branch_override = nil) - @default_branch_override = default_branch_override - dsl = R10K::Puppetfile::DSL.new(self) - dsl.instance_eval(puppetfile_contents, @puppetfile_path) - - validate_no_duplicate_names(@modules) + if default_branch_override && (default_branch_override != @default_branch_override) + logger.warn("Mismatch between passed and initialized default branch overrides, preferring passed value.") + @loader.default_branch_override = default_branch_override + end + + @loaded_content = @loader.load! @loaded = true - rescue SyntaxError, LoadError, ArgumentError, NameError => e - raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path}) + + @loaded_content end def loaded? @loaded end - # @param [Array] modules - def validate_no_duplicate_names(modules) - dupes = modules - .group_by { |mod| mod.name } - .select { |_, v| v.size > 1 } - .map(&:first) - unless dupes.empty? - msg = _('Puppetfiles cannot contain duplicate module names.') - msg += ' ' - msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') }) - raise R10K::Error.new(msg) - end + def modules + @loaded_content[:modules] end - # @param [String] forge - def set_forge(forge) - @forge = forge + # @see R10K::ModuleLoader::Puppetfile#add_module for upcoming signature changes + def add_module(name, args) + @loader.add_module(name, args) end - # @param [String] moduledir - def set_moduledir(moduledir) - @moduledir = if Pathname.new(moduledir).absolute? - moduledir - else - File.join(basedir, moduledir) - end + def set_moduledir(dir) + @loader.set_moduledir(dir) end - # @param [String] name - # @param [*Object] args - def add_module(name, args) - if args.is_a?(Hash) && install_path = args.delete(:install_path) - install_path = resolve_install_path(install_path) - validate_install_path(install_path, name) - else - install_path = @moduledir - end - - if args.is_a?(Hash) && @default_branch_override != nil - args[:default_branch] = @default_branch_override - end + def set_forge(forge) + @loader.set_forge(forge) + end - # Keep track of all the content this Puppetfile is managing to enable purging. - @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path) + def moduledir + @loader.moduledir + end - mod = R10K::Module.new(name, install_path, args, @environment) - mod.origin = 'Puppetfile' + def puppetfile_path + @loader.puppetfile_path + end - @managed_content[install_path] << mod.name - cachedir = mod.cachedir - @modules_by_vcs_cachedir[cachedir] ||= [] - @modules_by_vcs_cachedir[cachedir] << mod - @modules << mod + def environment=(env) + @loader.environment = env + @environment = env end include R10K::Util::Purgeable def managed_directories - self.load unless @loaded + self.load - dirs = @managed_content.keys - dirs.delete(real_basedir) - dirs + @loaded_content[:managed_directories] end # Returns an array of the full paths to all the content being managed. # @note This implements a required method for the Purgeable mixin # @return [Array] def desired_contents - self.load unless @loaded + self.load - @managed_content.flat_map do |install_path, modnames| - modnames.collect { |name| File.join(install_path, name) } - end + @loaded_content[:desired_contents] end def purge_exclusions - exclusions = managed_directories + self.load - if environment && environment.respond_to?(:desired_contents) - exclusions += environment.desired_contents - end - - exclusions + @loaded_content[:purge_exclusions] end def accept(visitor) pool_size = self.settings[:pool_size] if pool_size > 1 - concurrent_accept(visitor, pool_size) + R10K::ContentSynchronizer.concurrent_accept(modules, visitor, self, pool_size, logger) else - serial_accept(visitor) - end - end - - private - - def serial_accept(visitor) - visitor.visit(:puppetfile, self) do - modules.each do |mod| - mod.accept(visitor) - end + R10K::ContentSynchronizer.serial_accept(modules, visitor, self) end end - def concurrent_accept(visitor, pool_size) - logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size} - mods_queue = modules_queue(visitor) - thread_pool = pool_size.times.map { visitor_thread(visitor, mods_queue) } - thread_exception = nil - - # If any threads raise an exception the deployment is considered a failure. - # In that event clear the queue, wait for other threads to finish their - # current work, then re-raise the first exception caught. - begin - thread_pool.each(&:join) - rescue => e - logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message} - mods_queue.clear - thread_exception ||= e - retry - ensure - raise thread_exception unless thread_exception.nil? - end - end - - def modules_queue(visitor) - Queue.new.tap do |queue| - visitor.visit(:puppetfile, self) do - modules_by_cachedir = modules_by_vcs_cachedir.clone - modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || [] - - modules_without_vcs_cachedir.each {|mod| queue << Array(mod) } - modules_by_cachedir.values.each {|mods| queue << mods } - end - end - end - public :modules_queue - - def visitor_thread(visitor, mods_queue) - Thread.new do - begin - while mods = mods_queue.pop(true) do - mods.each {|mod| mod.accept(visitor) } - end - rescue ThreadError => e - logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id} - Thread.exit - rescue => e - Thread.main.raise(e) - end + def sync + pool_size = self.settings[:pool_size] + if pool_size > 1 + R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger) + else + R10K::ContentSynchronizer.serial_sync(modules) end end - def puppetfile_contents - File.read(@puppetfile_path) - end - - def resolve_install_path(path) - pn = Pathname.new(path) - - unless pn.absolute? - pn = Pathname.new(File.join(basedir, path)) - end - - # .cleanpath is as good as we can do without touching the filesystem. - # The .realpath methods will also choke if some of the intermediate - # paths are missing, even though we will create them later as needed. - pn.cleanpath.to_s - end - - def validate_install_path(path, modname) - unless /^#{Regexp.escape(real_basedir)}.*/ =~ path - raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}") - end - - true - end + private def real_basedir Pathname.new(basedir).cleanpath.to_s end - class DSL - # A barebones implementation of the Puppetfile DSL - # - # @api private - - def initialize(librarian) - @librarian = librarian - end - - def mod(name, args = nil) - @librarian.add_module(name, args) - end - - def forge(location) - @librarian.set_forge(location) - end - - def moduledir(location) - @librarian.set_moduledir(location) - end - - def method_missing(method, *args) - raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method} - end - end + DSL = R10K::ModuleLoader::Puppetfile::DSL end end diff -Nru r10k-3.7.0/lib/r10k/settings/container.rb r10k-4.0.0/lib/r10k/settings/container.rb --- r10k-3.7.0/lib/r10k/settings/container.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/settings/container.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,3 +1,4 @@ +require 'set' # Defines a collection for application settings # # This implements a hierarchical interface to application settings. Containers diff -Nru r10k-3.7.0/lib/r10k/settings/definition.rb r10k-4.0.0/lib/r10k/settings/definition.rb --- r10k-3.7.0/lib/r10k/settings/definition.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/settings/definition.rb 2023-08-01 15:06:51.000000000 +0000 @@ -90,7 +90,7 @@ def resolve if !@value.nil? @value - elsif @default + elsif !@default.nil? if @default == :inherit # walk all the way up to root, starting with grandparent ancestor = parent diff -Nru r10k-3.7.0/lib/r10k/settings.rb r10k-4.0.0/lib/r10k/settings.rb --- r10k-3.7.0/lib/r10k/settings.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/settings.rb 2023-08-01 15:06:51.000000000 +0000 @@ -19,6 +19,10 @@ def self.git_settings R10K::Settings::Collection.new(:git, [ + Definition.new(:default_ref, { + :desc => "User-defined default ref from which to deploy modules when not otherwise specified; nil unless configured via the r10k.yaml config.", + :default => nil}), + EnumDefinition.new(:provider, { :desc => "The Git provider to use. Valid values: 'shellgit', 'rugged'", :normalize => lambda { |input| input.to_sym }, @@ -37,6 +41,27 @@ Only used by the 'rugged' Git provider.", }), + Definition.new(:oauth_token, { + :desc => "The path to a token file for Git OAuth remotes. + Only used by the 'rugged' Git provider." + }), + + Definition.new(:github_app_id, { + :desc => "The Github App id for Git SSL remotes. + Only used by the 'rugged' Git provider." + }), + + Definition.new(:github_app_key, { + :desc => "The Github App private key for Git SSL remotes. + Only used by the 'rugged' Git provider." + }), + + Definition.new(:github_app_ttl, { + :desc => "The ttl expiration for SSL tokens. + Only used by the 'rugged' Git provider.", + :default => "120", + }), + URIDefinition.new(:proxy, { :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).", :default => :inherit, @@ -54,11 +79,35 @@ :default => :inherit, }), + Definition.new(:oauth_token, { + :desc => "The path to a token file for Git OAuth remotes. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + + Definition.new(:github_app_id, { + :desc => "The Github App id for Git SSL remotes. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + + Definition.new(:github_app_key, { + :desc => "The Github App private key for Git SSL remotes. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + + Definition.new(:github_app_ttl, { + :desc => "The ttl expiration for Git SSL tokens. + Only used by the 'rugged' Git provider.", + :default => :inherit + }), + URIDefinition.new(:proxy, { :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).", :default => :inherit, }), - + Definition.new(:ignore_branch_prefixes, { :desc => "Array of strings used to prefix branch names that will not be deployed as environments.", }), @@ -81,6 +130,20 @@ URIDefinition.new(:baseurl, { :desc => "The URL to the Puppet Forge to use for downloading modules." }), + + Definition.new(:authorization_token, { + :desc => "The token for Puppet Forge authorization. Leave blank for unauthorized or license-based connections." + }), + + Definition.new(:allow_puppetfile_override, { + :desc => "Whether to use `forge` declarations in the Puppetfile as an override of `baseurl`.", + :default => false, + :validate => lambda do |value| + unless !!value == value + raise ArgumentError, "`allow_puppetfile_override` can only be a boolean value, not '#{value}'" + end + end + }) ]) end @@ -111,7 +174,7 @@ end, }), - Definition.new(:purge_whitelist, { + Definition.new(:purge_allowlist, { :desc => "A list of filename patterns to be excluded from any purge operations. Patterns are matched relative to the root of each deployed environment, if you want a pattern to match recursively you need to use the '**' glob in your pattern. Basic shell style globs are supported.", :default => [], }), @@ -142,6 +205,47 @@ end end }), + Definition.new(:exclude_spec, { + :desc => "Whether or not to deploy the spec dir of a module. Defaults to true.", + :default => true, + :validate => lambda do |value| + unless !!value == value + raise ArgumentError, "`exclude_spec` can only be a boolean value, not '#{value}'" + end + end + })]) + end + + def self.logging_settings + R10K::Settings::Collection.new(:logging, [ + Definition.new(:level, { + desc: 'What logging level should R10k run on if not specified at runtime.', + validate: lambda do |value| + if R10K::Logging.parse_level(value).nil? + raise ArgumentError, "`level` must be a valid log level. + Valid levels are #{R10K::Logging::LOG_LEVELS.map(&:downcase).inspect}" + end + end + }), + + Definition.new(:outputs, { + desc: 'Additional log outputs to use.', + validate: lambda do |value| + unless value.is_a?(Array) + raise ArgumentError, "The `outputs` setting should be an array of outputs, not a #{value.class}" + end + end + }), + + Definition.new(:disable_default_stderr, { + desc: 'Disable the default stderr logging output', + default: false, + validate: lambda do |value| + unless !!value == value + raise ArgumentError, "`disable_default_stderr` can only be a boolean value, not '#{value}'" + end + end + }) ]) end @@ -161,7 +265,7 @@ }), Definition.new(:postrun, { - :desc => "The command r10k should run after deploying environments.", + :desc => "The command r10k should run after deploying environments or modules.", :validate => lambda do |value| if !value.is_a?(Array) raise ArgumentError, "The postrun setting should be an array of strings, not a #{value.class}" @@ -199,6 +303,8 @@ R10K::Settings.git_settings, R10K::Settings.deploy_settings, + + R10K::Settings.logging_settings ]) end end diff -Nru r10k-3.7.0/lib/r10k/source/base.rb r10k-4.0.0/lib/r10k/source/base.rb --- r10k-3.7.0/lib/r10k/source/base.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/source/base.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,8 +1,12 @@ +require 'r10k/logging' + # This class defines a common interface for source implementations. # # @since 1.3.0 class R10K::Source::Base + include R10K::Logging + # @!attribute [r] basedir # @return [String] The path this source will place environments in attr_reader :basedir @@ -31,10 +35,15 @@ # @option options [Boolean, String] :prefix If a String this becomes the prefix. # If true, will use the source name as the prefix. All sources should respect this option. # Defaults to false for no environment prefix. + # @option options [String] :strip_component If a string, this value will be + # removed from the beginning of each generated environment's name, if + # present. If the string is contained within two "/" characters, it will + # be treated as a regular expression. def initialize(name, basedir, options = {}) @name = name @basedir = Pathname.new(basedir).cleanpath.to_s @prefix = options.delete(:prefix) + @strip_component = options.delete(:strip_component) @puppetfile_name = options.delete(:puppetfile_name) @options = options end @@ -54,6 +63,16 @@ end + # Perform actions to reload environments after the `preload!`. Similar + # to preload!, and likely to include network queries and rerunning + # environment generation. + # + # @api public + # @abstract + # @return [void] + def reload! + end + # Enumerate the environments associated with this SVN source. # # @api public diff -Nru r10k-3.7.0/lib/r10k/source/git.rb r10k-4.0.0/lib/r10k/source/git.rb --- r10k-3.7.0/lib/r10k/source/git.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/source/git.rb 2023-08-01 15:06:51.000000000 +0000 @@ -11,8 +11,6 @@ # @since 1.3.0 class R10K::Source::Git < R10K::Source::Base - include R10K::Logging - R10K::Source.register(:git, self) # Register git as the default source R10K::Source.register(nil, self) @@ -95,18 +93,33 @@ end end + def reload! + @cache.sync! + @environments = generate_environments() + end + def generate_environments envs = [] - branch_names.each do |bn| - if bn.valid? - envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname, - {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name }) - elsif bn.correct? - logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: bn.name.inspect, corrected_env_name: bn.dirname} - envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname, - {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name}) - elsif bn.validate? - logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: bn.name.inspect} + environment_names.each do |en| + if en.valid? + envs << R10K::Environment::Git.new(en.name, + @basedir, + en.dirname, + {remote: remote, + ref: en.original_name, + puppetfile_name: puppetfile_name, + overrides: @options[:overrides]}) + elsif en.correct? + logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: en.name.inspect, corrected_env_name: en.dirname} + envs << R10K::Environment::Git.new(en.name, + @basedir, + en.dirname, + {remote: remote, + ref: en.original_name, + puppetfile_name: puppetfile_name, + overrides: @options[:overrides]}) + elsif en.validate? + logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: en.name.inspect} end end @@ -144,19 +157,22 @@ private - def branch_names - opts = {:prefix => @prefix, :invalid => @invalid_branches, :source => @name} - branches = @cache.branches + def environment_names + opts = {prefix: @prefix, + invalid: @invalid_branches, + source: @name, + strip_component: @strip_component} + branch_names = @cache.branches if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty? - branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes) + branch_names = filter_branches_by_regexp(branch_names, @ignore_branch_prefixes) end if @filter_command && !@filter_command.empty? - branches = filter_branches_by_command(branches, @filter_command) + branch_names = filter_branches_by_command(branch_names, @filter_command) end - branches.map do |branch| - R10K::Environment::Name.new(branch, opts) + branch_names.map do |branch_name| + R10K::Environment::Name.new(branch_name, opts) end end end diff -Nru r10k-3.7.0/lib/r10k/source/hash.rb r10k-4.0.0/lib/r10k/source/hash.rb --- r10k-3.7.0/lib/r10k/source/hash.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/source/hash.rb 2023-08-01 15:06:51.000000000 +0000 @@ -120,8 +120,6 @@ # class R10K::Source::Hash < R10K::Source::Base - include R10K::Logging - # @param hash [Hash] A hash to validate. # @return [Boolean] False if the hash is obviously invalid. A true return # means _maybe_ it's valid. @@ -152,8 +150,10 @@ R10K::Util::SymbolizeKeys.symbolize_keys!(opts) memo.merge({ name => opts.merge({ - :basedir => @basedir, - :dirname => R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname + basedir: @basedir, + dirname: R10K::Environment::Name.new(name, {prefix: @prefix, + source: @name, + strip_component: @strip_component}).dirname }) }) end @@ -168,7 +168,7 @@ def environments @environments ||= environments_hash.map do |name, hash| - R10K::Environment.from_hash(name, hash) + R10K::Environment.from_hash(name, hash.merge({overrides: @options[:overrides]})) end end diff -Nru r10k-3.7.0/lib/r10k/source/svn.rb r10k-4.0.0/lib/r10k/source/svn.rb --- r10k-3.7.0/lib/r10k/source/svn.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/source/svn.rb 2023-08-01 15:06:51.000000000 +0000 @@ -65,6 +65,10 @@ @ignore_branch_prefixes = options[:ignore_branch_prefixes] end + def reload! + @environments = generate_environments() + end + # Enumerate the environments associated with this SVN source. # # @return [Array] An array of environments created @@ -103,8 +107,6 @@ @environments.map {|env| env.dirname } end - include R10K::Logging - def filter_branches(branches, ignore_prefixes) filter = Regexp.new("^(#{ignore_prefixes.join('|')})") branches = branches.reject do |branch| @@ -121,7 +123,11 @@ def names_and_paths branches = [] - opts = {:prefix => @prefix, :correct => false, :validate => false, :source => @name} + opts = {prefix: @prefix, + correct: false, + validate: false, + source: @name, + strip_component: @strip_component} branches << [R10K::Environment::Name.new('production', opts), "#{@remote}/trunk"] additional_branch_names = @svn_remote.branches if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty? diff -Nru r10k-3.7.0/lib/r10k/source/yaml.rb r10k-4.0.0/lib/r10k/source/yaml.rb --- r10k-3.7.0/lib/r10k/source/yaml.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/source/yaml.rb 2023-08-01 15:06:51.000000000 +0000 @@ -7,7 +7,7 @@ begin contents = ::YAML.load_file(config) rescue => e - raise ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message} + raise R10K::ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message} end # Set the environments key for the parent class to consume diff -Nru r10k-3.7.0/lib/r10k/tarball.rb r10k-4.0.0/lib/r10k/tarball.rb --- r10k-3.7.0/lib/r10k/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/tarball.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,183 @@ +require 'fileutils' +require 'find' +require 'minitar' +require 'tempfile' +require 'uri' +require 'zlib' +require 'r10k/settings' +require 'r10k/settings/mixin' +require 'r10k/util/platform' +require 'r10k/util/cacheable' +require 'r10k/util/downloader' + +module R10K + class Tarball + + include R10K::Settings::Mixin + include R10K::Util::Cacheable + include R10K::Util::Downloader + + def_setting_attr :proxy # Defaults to global proxy setting + def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir + + # @!attribute [rw] name + # @return [String] The tarball's name + attr_accessor :name + + # @!attribute [rw] source + # @return [String] The tarball's source + attr_accessor :source + + # @!attribute [rw] checksum + # @return [String] The tarball's expected sha256 digest + attr_accessor :checksum + + # @param name [String] The name of the tarball content + # @param source [String] The source for the tarball content + # @param checksum [String] The sha256 digest of the tarball content + def initialize(name, source, checksum: nil) + @name = name + @source = source + @checksum = checksum + + # At this time, the only checksum type supported is sha256. In the future, + # we may decide to support other algorithms if a use case arises. TBD. + checksum_algorithm = :SHA256 + end + + # @return [String] Directory. Where the cache_basename file will be created. + def cache_dirname + File.join(settings[:cache_root], 'tarball') + end + + # The final cache_path should match one of the templates: + # + # - {cachedir}/{checksum}.tar.gz + # - {cachedir}/{source}.tar.gz + # + # @return [String] File. The full file path the tarball will be cached to. + def cache_path + File.join(cache_dirname, cache_basename) + end + + # @return [String] The basename of the tarball cache file. + def cache_basename + if checksum.nil? + sanitized_dirname(source) + '.tar.gz' + else + checksum + '.tar.gz' + end + end + + # Extract the cached tarball to the target directory. + # + # @param target_dir [String] Where to unpack the tarball + def unpack(target_dir) + file = File.open(cache_path, 'rb') + reader = Zlib::GzipReader.new(file) + begin + Minitar.unpack(reader, target_dir) + ensure + reader.close + end + end + + # @param target_dir [String] The directory to check if is in sync with the + # tarball content + # @param ignore_untracked_files [Boolean] If true, consider the target + # dir to be in sync as long as all tracked content matches. + # + # @return [Boolean] + def insync?(target_dir, ignore_untracked_files: false) + target_tree_entries = Find.find(target_dir).map(&:to_s) - [target_dir] + each_tarball_entry do |entry| + found = target_tree_entries.delete(File.join(target_dir, entry.full_name.chomp('/'))) + return false if found.nil? + next if entry.directory? + return false unless file_digest(found) == reader_digest(entry) + end + + if ignore_untracked_files + # We wouldn't have gotten this far if there were discrepancies in + # tracked content + true + else + # If there are still files in target_tree_entries, then there is + # untracked content present in the target tree. If not, we're in sync. + target_tree_entries.empty? + end + end + + # Download the tarball from @source to @cache_path + def get + Tempfile.open(cache_basename) do |tempfile| + tempfile.binmode + src_uri = URI.parse(source) + + temp_digest = case src_uri.scheme + when 'file', nil + copy(src_uri.path, tempfile) + when %r{^[a-z]$} # Windows drive letter + copy(src_uri.to_s, tempfile) + when %r{^https?$} + download(src_uri, tempfile) + else + raise "Unexpected source scheme #{src_uri.scheme}" + end + + # Verify the download + unless (checksum == temp_digest) || checksum.nil? + raise 'Downloaded file does not match checksum' + end + + # Move the download to cache_path + FileUtils::mkdir_p(cache_dirname) + begin + FileUtils.mv(tempfile.path, cache_path) + rescue Errno::EACCES + # It may be the case that permissions don't permit moving the file + # into place, but do permit overwriting an existing in-place file. + FileUtils.cp(tempfile.path, cache_path) + end + end + end + + # Checks the cached tarball's digest against the expected checksum. Returns + # false if no cached file is present. If the tarball has no expected + # checksum, any cached file is assumed to be valid. + # + # @return [Boolean] + def cache_valid? + return false unless File.exist?(cache_path) + return true if checksum.nil? + checksum == file_digest(cache_path) + end + + # List all of the files contained in the tarball and their paths. This is + # useful for implementing R10K::Purgable + # + # @return [Array] A normalized list of file paths contained in the archive + def paths + names = Array.new + each_tarball_entry { |entry| names << Pathname.new(entry).cleanpath.to_s } + names - ['.'] + end + + def cache_checksum + raise R10K::Error, _("Cache not present at %{path}") % {path: cache_path} unless File.exist?(cache_path) + file_digest(cache_path) + end + + private + + def each_tarball_entry(&block) + File.open(cache_path, 'rb') do |file| + Zlib::GzipReader.wrap(file) do |reader| + Archive::Tar::Minitar::Input.each_entry(reader) do |entry| + yield entry + end + end + end + end + end +end diff -Nru r10k-3.7.0/lib/r10k/util/cacheable.rb r10k-4.0.0/lib/r10k/util/cacheable.rb --- r10k-3.7.0/lib/r10k/util/cacheable.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/util/cacheable.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,31 @@ +module R10K + module Util + + # Utility mixin for classes that need to implement caches + # + # @abstract Classes using this mixin need to implement {#managed_directory} and + # {#desired_contents} + module Cacheable + + # Provide a default cachedir location. This is consumed by R10K::Settings + # for appropriate global default values. + # + # @return [String] Path to the default cache directory + def self.default_cachedir(basename = 'cache') + if R10K::Util::Platform.windows? + File.join(ENV['LOCALAPPDATA'], 'r10k', basename) + else + File.join(ENV['HOME'] || '/root', '.r10k', basename) + end + end + + # Reformat a string into something that can be used as a directory + # + # @param string [String] An identifier to create a sanitized dirname for + # @return [String] A sanitized dirname for the given string + def sanitized_dirname(string) + string.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-') + end + end + end +end diff -Nru r10k-3.7.0/lib/r10k/util/cleaner.rb r10k-4.0.0/lib/r10k/util/cleaner.rb --- r10k-3.7.0/lib/r10k/util/cleaner.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/util/cleaner.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,21 @@ +require 'r10k/logging' +require 'r10k/util/purgeable' + +module R10K + module Util + class Cleaner + + include R10K::Logging + include R10K::Util::Purgeable + + attr_reader :managed_directories, :desired_contents, :purge_exclusions + + def initialize(managed_directories, desired_contents, purge_exclusions = []) + @managed_directories = managed_directories + @desired_contents = desired_contents + @purge_exclusions = purge_exclusions + end + + end + end +end diff -Nru r10k-3.7.0/lib/r10k/util/downloader.rb r10k-4.0.0/lib/r10k/util/downloader.rb --- r10k-3.7.0/lib/r10k/util/downloader.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/lib/r10k/util/downloader.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,134 @@ +require 'digest' +require 'net/http' + +module R10K + module Util + + # Utility mixin for classes that need to download files + module Downloader + + # Downloader objects need to checksum downloaded or saved content. The + # algorithm used to perform this checksumming (and therefore the kinds of + # checksums returned by various methods) is reported by this method. + # + # @return [Symbol] The checksum algorithm the downloader uses + def checksum_algorithm + @checksum_algorithm ||= :SHA256 + end + + private + + # Set the checksum algorithm the downloader should use. It should be a + # symbol, and a valid Ruby 'digest' library algorithm. The default is + # :SHA256. + # + # @param algorithm [Symbol] The checksum algorithm the downloader should use + def checksum_algorithm=(algorithm) + @checksum_algorithm = algorithm + end + + CHUNK_SIZE = 64 * 1024 # 64 kb + + # @param src_uri [URI] The URI to download from + # @param dst_file [String] The file or path to save to + # @return [String] The downloaded file's hex digest + def download(src_uri, dst_file) + digest = Digest(checksum_algorithm).new + http_get(src_uri) do |resp| + File.open(dst_file, 'wb') do |output_stream| + resp.read_body do |chunk| + output_stream.write(chunk) + digest.update(chunk) + end + end + end + + digest.hexdigest + end + + # @param src_file The file or path to copy from + # @param dst_file The file or path to copy to + # @return [String] The copied file's sha256 hex digest + def copy(src_file, dst_file) + digest = Digest(checksum_algorithm).new + File.open(src_file, 'rb') do |input_stream| + File.open(dst_file, 'wb') do |output_stream| + until input_stream.eof? + chunk = input_stream.read(CHUNK_SIZE) + output_stream.write(chunk) + digest.update(chunk) + end + end + end + + digest.hexdigest + end + + # Start a Net::HTTP::Get connection, then yield the Net::HTTPSuccess object + # to the caller's block. Follow redirects if Net::HTTPRedirection responses + # are encountered, and use a proxy if directed. + # + # @param uri [URI] The URI to download the file from + # @param redirect_limit [Integer] How many redirects to permit before failing + # @param proxy [URI, String] The URI to use as a proxy + def http_get(uri, redirect_limit: 10, proxy: nil, &block) + raise "HTTP redirect too deep" if redirect_limit.zero? + + session = Net::HTTP.new(uri.host, uri.port, *proxy_to_array(proxy)) + session.use_ssl = true if uri.scheme == 'https' + session.start + + begin + session.request_get(uri) do |response| + case response + when Net::HTTPRedirection + redirect = response['location'] + session.finish + return http_get(URI.parse(redirect), redirect_limit: redirect_limit - 1, proxy: proxy, &block) + when Net::HTTPSuccess + yield response + else + raise "Unexpected response code #{response.code}: #{response}" + end + end + ensure + session.finish if session.active? + end + end + + # Helper method to translate a proxy URI to array arguments for + # Net::HTTP#new. A nil argument returns nil array elements. + def proxy_to_array(proxy_uri) + if proxy_uri + px = proxy_uri.is_a?(URI) ? proxy_uri : URI.parse(proxy_uri) + [px.host, px.port, px.user, px.password] + else + [nil, nil, nil, nil] + end + end + + # Return the sha256 digest of the file at the given path + # + # @param path [String] The path to the file + # @return [String] The file's sha256 hex digest + def file_digest(path) + File.open(path) do |file| + reader_digest(file) + end + end + + # Return the sha256 digest of the readable data + # + # @param reader [String] An object that responds to #read + # @return [String] The read data's sha256 hex digest + def reader_digest(reader) + digest = Digest(checksum_algorithm).new + while chunk = reader.read(CHUNK_SIZE) + digest.update(chunk) + end + + digest.hexdigest + end + end + end +end diff -Nru r10k-3.7.0/lib/r10k/util/purgeable.rb r10k-4.0.0/lib/r10k/util/purgeable.rb --- r10k-3.7.0/lib/r10k/util/purgeable.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/util/purgeable.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,3 +1,5 @@ +require 'r10k/logging' + require 'fileutils' module R10K @@ -9,6 +11,14 @@ # {#desired_contents} module Purgeable + include R10K::Logging + + HIDDEN_FILE = /\.[^.]+/ + + FN_MATCH_OPTS = File::FNM_PATHNAME | File::FNM_DOTMATCH + + # @deprecated + # # @!method logger # @abstract Including classes must provide a logger method # @return [Log4r::Logger] @@ -38,23 +48,79 @@ end end + # @deprecated Unused helper function + # # @return [Array] Directory contents that are expected but not present def pending_contents(recurse) desired_contents - current_contents(recurse) end + def matches?(test, path) + if test == path + true + elsif File.fnmatch?(test, path, FN_MATCH_OPTS) + true + else + false + end + end + + # A method to collect potentially purgeable content without searching into + # ignored directories when recursively searching. + # + # @param dir [String, Pathname] The directory to search for purgeable content + # @param exclusion_gobs [Array] A list of file paths or File globs + # to exclude from recursion (These are generated by the classes that + # mix this module into them and are typically programatically generated) + # @param allowed_gobs [Array] A list of file paths or File globs to exclude + # from recursion (These are passed in by the caller of purge! and typically + # are user supplied configuration values) + # @param desireds_not_to_recurse_into [Array] A list of file paths not to + # recurse into. These are programatically generated, these exist to maintain + # backwards compatibility with previous implementations that used File.globs + # for "recursion", ie "**/{*,.[^.]*}" which would not recurse into dot directories. + # @param recurse [Boolean] Whether or not to recurse into child directories that do + # not match other filters. + # + # @return [Array] Contents which may be purged. + def potentially_purgeable(dir, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse) + children = Pathname.new(dir).children.reject do |path| + path = path.to_s + + if exclusion_match = exclusion_globs.find { |exclusion| matches?(exclusion, path) } + logger.debug2 _("Not purging %{path} due to internal exclusion match: %{exclusion_match}") % {path: path, exclusion_match: exclusion_match} + elsif allowlist_match = allowed_globs.find { |allowed| matches?(allowed, path) } + logger.debug _("Not purging %{path} due to whitelist match: %{allowlist_match}") % {path: path, allowlist_match: allowlist_match} + else + desired_match = desireds_not_to_recurse_into.grep(path).first + end + + !!exclusion_match || !!allowlist_match || !!desired_match + end + + children.flat_map do |child| + if File.directory?(child) && !File.symlink?(child) && recurse + potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse) << child.to_s + else + child.to_s + end + end + end + # @return [Array] Directory contents that are present but not expected def stale_contents(recurse, exclusions, whitelist) - fn_match_opts = File::FNM_PATHNAME | File::FNM_DOTMATCH + dirs = self.managed_directories + desireds = self.desired_contents + hidden_desireds, regular_desireds = desireds.partition do |desired| + HIDDEN_FILE.match(File.basename(desired)) + end - (current_contents(recurse) - desired_contents).reject do |item| - if exclusion_match = exclusions.find { |ex_item| (ex_item == item) || File.fnmatch?(ex_item, item, fn_match_opts) } - logger.debug2 _("Not purging %{item} due to internal exclusion match: %{exclusion_match}") % {item: item, exclusion_match: exclusion_match} - elsif whitelist_match = whitelist.find { |wl_item| (wl_item == item) || File.fnmatch?(wl_item, item, fn_match_opts) } - logger.debug _("Not purging %{item} due to whitelist match: %{whitelist_match}") % {item: item, whitelist_match: whitelist_match} - end + initial_purgelist = dirs.flat_map do |dir| + potentially_purgeable(dir, exclusions, whitelist, hidden_desireds, recurse) + end - !!exclusion_match || !!whitelist_match + initial_purgelist.reject do |path| + regular_desireds.any? { |desired| matches?(desired, path) } end end diff -Nru r10k-3.7.0/lib/r10k/util/setopts.rb r10k-4.0.0/lib/r10k/util/setopts.rb --- r10k-3.7.0/lib/r10k/util/setopts.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/util/setopts.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,3 +1,5 @@ +require 'r10k/logging' + module R10K module Util @@ -7,6 +9,10 @@ # supports Ruby 1.8.7+ we cannot use that functionality. module Setopts + class Ignore; end + + include R10K::Logging + private # @param opts [Hash] @@ -31,22 +37,39 @@ # setopts(opts, allowed) # @trace # => nil # - def setopts(opts, allowed) + def setopts(opts, allowed, raise_on_unhandled: true) + processed_vars = {} opts.each_pair do |key, value| if allowed.key?(key) - rhs = allowed[key] - case rhs - when NilClass, FalseClass - # Ignore nil options - when :self, TrueClass - # tr here is because instance variables cannot have hyphens in their names. - instance_variable_set("@#{key}".tr('-','_').to_sym, value) - else - # tr here same as previous - instance_variable_set("@#{rhs}".tr('-','_').to_sym, value) + # Ignore nil options and explicit ignore param + next unless rhs = allowed[key] + next if rhs == ::R10K::Util::Setopts::Ignore + + var = case rhs + when :self, TrueClass + # tr here is because instance variables cannot have hyphens in their names. + "@#{key}".tr('-','_').to_sym + else + # tr here same as previous + "@#{rhs}".tr('-','_').to_sym + end + + if processed_vars.include?(var) + # This should be a raise, but that would be a behavior change and + # should happen on a SemVer boundry. + logger.warn _("%{class_name} parameters '%{a}' and '%{b}' conflict. Specify one or the other, but not both" \ + % {class_name: self.class.name, a: processed_vars[var], b: key}) end + + instance_variable_set(var, value) + processed_vars[var] = key else - raise ArgumentError, _("%{class_name} cannot handle option '%{key}'") % {class_name: self.class.name, key: key} + err_str = _("%{class_name} cannot handle option '%{key}'") % {class_name: self.class.name, key: key} + if raise_on_unhandled + raise ArgumentError, err_str + else + logger.warn(err_str) + end end end end diff -Nru r10k-3.7.0/lib/r10k/util/subprocess.rb r10k-4.0.0/lib/r10k/util/subprocess.rb --- r10k-3.7.0/lib/r10k/util/subprocess.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/util/subprocess.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,3 +1,4 @@ +require 'r10k/logging' require 'r10k/util/platform' module R10K diff -Nru r10k-3.7.0/lib/r10k/version.rb r10k-4.0.0/lib/r10k/version.rb --- r10k-3.7.0/lib/r10k/version.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/lib/r10k/version.rb 2023-08-01 15:06:51.000000000 +0000 @@ -2,5 +2,5 @@ # When updating to a new major (X) or minor (Y) version, include `#major` or # `#minor` (respectively) in your commit message to trigger the appropriate # release. Otherwise, a new patch (Z) version will be released. - VERSION = '3.7.0' + VERSION = '4.0.0' end diff -Nru r10k-3.7.0/locales/r10k.pot r10k-4.0.0/locales/r10k.pot --- r10k-3.7.0/locales/r10k.pot 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/locales/r10k.pot 2023-08-01 15:06:51.000000000 +0000 @@ -1,16 +1,16 @@ # SOME DESCRIPTIVE TITLE. -# Copyright (C) 2020 Puppet, Inc. +# Copyright (C) 2023 Puppet, Inc. # This file is distributed under the same license as the r10k package. -# FIRST AUTHOR , 2020. +# FIRST AUTHOR , 2023. # #, fuzzy msgid "" msgstr "" -"Project-Id-Version: r10k 3.4.1-57-g2eb088a\n" +"Project-Id-Version: r10k 3.9.3-373-g117587fb\n" "\n" "Report-Msgid-Bugs-To: docs@puppetlabs.com\n" -"POT-Creation-Date: 2020-07-22 16:41+0000\n" -"PO-Revision-Date: 2020-07-22 16:41+0000\n" +"POT-Creation-Date: 2023-07-26 22:34+0000\n" +"PO-Revision-Date: 2023-07-26 22:34+0000\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" @@ -19,80 +19,88 @@ "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n" -#: ../lib/r10k/action/deploy/deploy_helpers.rb:12 ../lib/r10k/settings/loader.rb:63 +#: ../lib/r10k/action/deploy/deploy_helpers.rb:16 ../lib/r10k/settings/loader.rb:63 msgid "No configuration file given, no config file found in current directory, and no global config present" msgstr "" -#: ../lib/r10k/action/deploy/deploy_helpers.rb:26 +#: ../lib/r10k/action/deploy/deploy_helpers.rb:30 msgid "Making changes to deployed environments has been administratively disabled." msgstr "" -#: ../lib/r10k/action/deploy/deploy_helpers.rb:27 +#: ../lib/r10k/action/deploy/deploy_helpers.rb:31 msgid "Reason: %{write_lock}" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:57 +#: ../lib/r10k/action/deploy/environment.rb:99 msgid "Environment(s) \\'%{environments}\\' cannot be found in any source and will not be deployed." msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:85 +#: ../lib/r10k/action/deploy/environment.rb:119 +msgid "Executing postrun command." +msgstr "" + +#: ../lib/r10k/action/deploy/environment.rb:133 msgid "Environment %{env_dir} does not match environment name filter, skipping" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:93 +#: ../lib/r10k/action/deploy/environment.rb:141 msgid "Deploying environment %{env_path}" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:96 +#: ../lib/r10k/action/deploy/environment.rb:144 msgid "Environment %{env_dir} is now at %{env_signature}" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:100 +#: ../lib/r10k/action/deploy/environment.rb:148 msgid "Environment %{env_dir} is new, updating all modules" msgstr "" -#: ../lib/r10k/action/deploy/environment.rb:143 -msgid "Deploying %{origin} content %{path}" +#: ../lib/r10k/action/deploy/module.rb:82 +msgid "Running postrun command for environments: %{envs_to_run}." msgstr "" -#: ../lib/r10k/action/deploy/module.rb:49 -msgid "Only updating modules in environment %{opt_env} skipping environment %{env_path}" +#: ../lib/r10k/action/deploy/module.rb:92 +msgid "No environments were modified, not executing postrun command." msgstr "" -#: ../lib/r10k/action/deploy/module.rb:51 +#: ../lib/r10k/action/deploy/module.rb:104 +msgid "Only updating modules in environment(s) %{opt_env} skipping environment %{env_path}" +msgstr "" + +#: ../lib/r10k/action/deploy/module.rb:106 msgid "Updating modules %{modules} in environment %{env_path}" msgstr "" -#: ../lib/r10k/action/deploy/module.rb:63 -msgid "Deploying module %{mod_path}" +#: ../lib/r10k/action/puppetfile/check.rb:25 +msgid "Syntax OK" msgstr "" -#: ../lib/r10k/action/deploy/module.rb:70 -msgid "Only updating modules %{modules}, skipping module %{mod_name}" +#: ../lib/r10k/action/runner.rb:63 ../lib/r10k/deployment/config.rb:42 +msgid "Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'" msgstr "" -#: ../lib/r10k/action/puppetfile/check.rb:14 -msgid "Syntax OK" +#: ../lib/r10k/action/runner.rb:105 +msgid "Reading configuration from %{config_path}" msgstr "" -#: ../lib/r10k/action/puppetfile/install.rb:30 -msgid "Updating module %{mod_path}" +#: ../lib/r10k/action/runner.rb:108 +msgid "No config file explicitly given and no default config file could be found, default settings will be used." msgstr "" -#: ../lib/r10k/action/puppetfile/install.rb:33 -msgid "Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." +#: ../lib/r10k/content_synchronizer.rb:33 +msgid "Updating modules with %{pool_size} threads" msgstr "" -#: ../lib/r10k/action/runner.rb:53 ../lib/r10k/deployment/config.rb:42 -msgid "Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'" +#: ../lib/r10k/content_synchronizer.rb:46 +msgid "Error during concurrent deploy of a module: %{message}" msgstr "" -#: ../lib/r10k/action/runner.rb:86 -msgid "Reading configuration from %{config_path}" +#: ../lib/r10k/content_synchronizer.rb:86 +msgid "Module %{mod_name} failed to synchronize due to %{message}" msgstr "" -#: ../lib/r10k/action/runner.rb:89 -msgid "No config file explicitly given and no default config file could be found, default settings will be used." +#: ../lib/r10k/content_synchronizer.rb:92 +msgid "Module thread %{id} exiting: %{message}" msgstr "" #: ../lib/r10k/deployment.rb:90 @@ -103,16 +111,20 @@ msgid "Unable to load sources; the supplied configuration does not define the 'sources' key" msgstr "" -#: ../lib/r10k/environment/base.rb:61 ../lib/r10k/environment/base.rb:77 ../lib/r10k/environment/base.rb:86 ../lib/r10k/source/base.rb:64 +#: ../lib/r10k/environment/base.rb:89 ../lib/r10k/environment/base.rb:105 ../lib/r10k/environment/base.rb:114 ../lib/r10k/source/base.rb:83 msgid "%{class} has not implemented method %{method}" msgstr "" -#: ../lib/r10k/environment/with_modules.rb:104 -msgid "Puppetfile cannot contain module names defined by environment %{name}" +#: ../lib/r10k/environment/name.rb:83 +msgid "Improper configuration value given for strip_component setting in %{src} source. Value must be a string, a /regex/, false, or omitted. Got \"%{val}\" (%{type})" msgstr "" -#: ../lib/r10k/environment/with_modules.rb:106 -msgid "Remove the conflicting definitions of the following modules: %{conflicts}" +#: ../lib/r10k/environment/with_modules.rb:60 +msgid "Environment and %{src} both define the \"%{name}\" module" +msgstr "" + +#: ../lib/r10k/environment/with_modules.rb:71 +msgid "Unexpected value for `module_conflicts` setting in %{env} environment: %{val}" msgstr "" #: ../lib/r10k/feature.rb:27 @@ -139,19 +151,19 @@ msgid "Proc %{block} for feature %{name} returned %{output}" msgstr "" -#: ../lib/r10k/forge/module_release.rb:196 +#: ../lib/r10k/forge/module_release.rb:197 msgid "Unpacking %{tarball_cache_path} to %{target_dir} (with tmpdir %{tmp_path})" msgstr "" -#: ../lib/r10k/forge/module_release.rb:198 +#: ../lib/r10k/forge/module_release.rb:199 msgid "Valid files unpacked: %{valid_files}" msgstr "" -#: ../lib/r10k/forge/module_release.rb:200 +#: ../lib/r10k/forge/module_release.rb:201 msgid "These files existed in the module's tar file, but are invalid filetypes and were not unpacked: %{invalid_files}" msgstr "" -#: ../lib/r10k/forge/module_release.rb:203 +#: ../lib/r10k/forge/module_release.rb:204 msgid "Symlinks are unsupported and were not unpacked from the module tarball. %{release_slug} contained these ignored symlinks: %{symlinks}" msgstr "" @@ -187,11 +199,11 @@ msgid "Cannot write %{file}; parent directory does not exist" msgstr "" -#: ../lib/r10k/git/cache.rb:55 +#: ../lib/r10k/git/cache.rb:57 msgid "%{class}#path is deprecated; use #git_dir" msgstr "" -#: ../lib/r10k/git/cache.rb:84 +#: ../lib/r10k/git/cache.rb:86 msgid "Creating new git cache for %{remote}" msgstr "" @@ -207,39 +219,99 @@ msgid "Rugged versions prior to 0.24.0 do not support pruning stale branches during fetch, please upgrade your \\'rugged\\' gem. (Current version is: %{version})" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:24 +#: ../lib/r10k/git/rugged/base_repository.rb:24 +msgid "Unable to resolve %{pattern}: %{e} " +msgstr "" + +#: ../lib/r10k/git/rugged/base_repository.rb:69 +msgid "Remote URL is different from cache, updating %{orig} to %{update}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:28 msgid "Authentication failed for Git remote %{url}." msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:48 +#: ../lib/r10k/git/rugged/credentials.rb:52 msgid "Using per-repository private key %{key} for URL %{url}" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:51 +#: ../lib/r10k/git/rugged/credentials.rb:55 msgid "URL %{url} has no per-repository private key using '%{key}'." msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:53 +#: ../lib/r10k/git/rugged/credentials.rb:57 msgid "Git remote %{url} uses the SSH protocol but no private key was given" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:57 +#: ../lib/r10k/git/rugged/credentials.rb:61 msgid "Unable to use SSH key auth for %{url}: private key %{private_key} is missing or unreadable" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:80 +#: ../lib/r10k/git/rugged/credentials.rb:102 +msgid "Using OAuth token from stdin for URL %{url}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:105 +msgid "Using OAuth token from %{token_path} for URL %{url}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:107 +msgid "%{path} is missing or unreadable, cannot load OAuth token" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:111 +msgid "Supplied OAuth token contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:135 msgid "URL %{url} includes the username %{username}, using that user for authentication." msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:83 +#: ../lib/r10k/git/rugged/credentials.rb:138 msgid "URL %{url} did not specify a user, using %{user} from configuration" msgstr "" -#: ../lib/r10k/git/rugged/credentials.rb:86 +#: ../lib/r10k/git/rugged/credentials.rb:141 msgid "URL %{url} did not specify a user, using current user %{user}" msgstr "" -#: ../lib/r10k/git/rugged/thin_repository.rb:85 ../lib/r10k/git/shellgit/thin_repository.rb:65 +#: ../lib/r10k/git/rugged/credentials.rb:148 +msgid "Github App id contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:149 +msgid "Github App token ttl contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:150 +msgid "Github App key is missing or unreadable" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:155 +msgid "Github App key is not a valid SSL private key" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:158 +msgid "Github App key is not a valid SSL key" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:161 +msgid "Using Github App id %{app_id} with SSL key from %{key_path}" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:179 +msgid "Error using private key to get Github App access token from url" +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:200 +msgid "Github App token contains invalid characters." +msgstr "" + +#: ../lib/r10k/git/rugged/credentials.rb:202 +msgid "Github App token generated, expires at: %{expire}" +msgstr "" + +#: ../lib/r10k/git/rugged/thin_repository.rb:92 ../lib/r10k/git/shellgit/thin_repository.rb:69 msgid "Updated repo %{path} to include alternate object db path %{objects_dir}" msgstr "" @@ -251,39 +323,39 @@ msgid "Fetching remote '%{remote}' at %{path}" msgstr "" -#: ../lib/r10k/git/rugged/working_repository.rb:125 ../lib/r10k/git/shellgit/working_repository.rb:100 +#: ../lib/r10k/git/rugged/working_repository.rb:131 ../lib/r10k/git/shellgit/working_repository.rb:101 msgid "Found local modifications in %{file_path}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:40 +#: ../lib/r10k/git/stateful_repository.rb:45 msgid "Unable to sync repo to unresolvable ref '%{ref}'" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:47 +#: ../lib/r10k/git/stateful_repository.rb:53 msgid "Cloning %{repo_path} and checking out %{ref}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:50 +#: ../lib/r10k/git/stateful_repository.rb:56 msgid "Replacing %{repo_path} and checking out %{ref}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:54 ../lib/r10k/git/stateful_repository.rb:59 +#: ../lib/r10k/git/stateful_repository.rb:60 ../lib/r10k/git/stateful_repository.rb:65 msgid "Updating %{repo_path} to %{ref}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:58 +#: ../lib/r10k/git/stateful_repository.rb:64 msgid "Overwriting local modifications to %{repo_path}" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:62 +#: ../lib/r10k/git/stateful_repository.rb:68 msgid "Skipping %{repo_path} due to local modifications" msgstr "" -#: ../lib/r10k/git/stateful_repository.rb:65 +#: ../lib/r10k/git/stateful_repository.rb:72 msgid "%{repo_path} is already at Git ref %{ref}" msgstr "" -#: ../lib/r10k/initializers.rb:30 +#: ../lib/r10k/initializers.rb:31 msgid "the purgedirs key in r10k.yaml is deprecated. it is currently ignored." msgstr "" @@ -295,31 +367,47 @@ msgid "No class registered for %{key}" msgstr "" -#: ../lib/r10k/logging.rb:60 +#: ../lib/r10k/logging.rb:73 ../lib/r10k/logging.rb:100 ../lib/r10k/logging.rb:109 msgid "Invalid log level '%{val}'. Valid levels are %{log_levels}" msgstr "" -#: ../lib/r10k/module.rb:29 +#: ../lib/r10k/module.rb:45 msgid "Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)" msgstr "" -#: ../lib/r10k/module/base.rb:110 +#: ../lib/r10k/module/base.rb:120 +msgid "Deploying module to %{path}" +msgstr "" + +#: ../lib/r10k/module/base.rb:123 +msgid "Only updating modules %{modules}, skipping module %{name}" +msgstr "" + +#: ../lib/r10k/module/base.rb:179 msgid "Module name (%{title}) must match either 'modulename' or 'owner/modulename'" msgstr "" -#: ../lib/r10k/module/forge.rb:70 ../lib/r10k/module/forge.rb:99 -msgid "The module %{title} does not exist on %{url}." +#: ../lib/r10k/module/definition.rb:28 +msgid "Not updating module %{name}, assuming content unchanged" +msgstr "" + +#: ../lib/r10k/module/forge.rb:50 +msgid "Module version %{ver} is not a valid Forge module version" +msgstr "" + +#: ../lib/r10k/module/forge.rb:98 +msgid "The module %{title} does not appear to have any published releases, cannot determine latest version." msgstr "" -#: ../lib/r10k/module/forge.rb:174 -msgid "Forge module names must match 'owner/modulename'" +#: ../lib/r10k/module/forge.rb:101 ../lib/r10k/module/forge.rb:130 +msgid "The module %{title} does not exist on %{url}." msgstr "" -#: ../lib/r10k/module/git.rb:97 -msgid "Unhandled options %{unhandled} specified for %{class}" +#: ../lib/r10k/module/git.rb:78 +msgid "Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available." msgstr "" -#: ../lib/r10k/module/local.rb:34 +#: ../lib/r10k/module/local.rb:37 msgid "Module %{title} is a local module, always indicating synced." msgstr "" @@ -327,39 +415,43 @@ msgid "Could not read metadata.json" msgstr "" -#: ../lib/r10k/puppetfile.rb:57 +#: ../lib/r10k/module_loader/puppetfile.rb:68 msgid "Using Puppetfile '%{puppetfile}'" msgstr "" -#: ../lib/r10k/puppetfile.rb:71 -msgid "Puppetfile %{path} missing or unreadable" +#: ../lib/r10k/module_loader/puppetfile.rb:69 +msgid "Using moduledir '%{moduledir}'" msgstr "" -#: ../lib/r10k/puppetfile.rb:84 +#: ../lib/r10k/module_loader/puppetfile.rb:91 msgid "Failed to evaluate %{path}" msgstr "" -#: ../lib/r10k/puppetfile.rb:98 -msgid "Puppetfiles cannot contain duplicate module names." +#: ../lib/r10k/module_loader/puppetfile.rb:108 +msgid "Unable to preload Puppetfile because of %{msg}" msgstr "" -#: ../lib/r10k/puppetfile.rb:100 -msgid "Remove the duplicates of the following modules: %{dupes}" +#: ../lib/r10k/module_loader/puppetfile.rb:126 +msgid "Using Forge from Puppetfile: %{forge}" msgstr "" -#: ../lib/r10k/puppetfile.rb:192 -msgid "Updating modules with %{pool_size} threads" +#: ../lib/r10k/module_loader/puppetfile.rb:129 +msgid "Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}." msgstr "" -#: ../lib/r10k/puppetfile.rb:203 -msgid "Error during concurrent deploy of a module: %{message}" +#: ../lib/r10k/module_loader/puppetfile.rb:193 ../lib/r10k/puppetfile.rb:104 +msgid "Puppetfile %{path} missing or unreadable" msgstr "" -#: ../lib/r10k/puppetfile.rb:225 -msgid "Module thread %{id} exiting: %{message}" +#: ../lib/r10k/module_loader/puppetfile.rb:235 +msgid "Puppetfiles cannot contain duplicate module names." msgstr "" -#: ../lib/r10k/puppetfile.rb:282 +#: ../lib/r10k/module_loader/puppetfile.rb:237 +msgid "Remove the duplicates of the following modules: %{dupes}" +msgstr "" + +#: ../lib/r10k/module_loader/puppetfile/dsl.rb:37 msgid "unrecognized declaration '%{method}'" msgstr "" @@ -371,7 +463,7 @@ msgid "Validation failed for settings group" msgstr "" -#: ../lib/r10k/settings/container.rb:91 +#: ../lib/r10k/settings/container.rb:92 msgid "Key %{key} is not a valid key" msgstr "" @@ -415,10 +507,6 @@ msgid "Couldn't load config file: %{error_msg}" msgstr "" -#: ../lib/r10k/settings/loader.rb:73 -msgid "File exists at #{path} but doesn't contain any YAML" -msgstr "" - #: ../lib/r10k/settings/uri_definition.rb:12 msgid "Setting %{name} requires a URL but '%{value}' could not be parsed as a URL" msgstr "" @@ -442,27 +530,27 @@ "Returned: %{data}" msgstr "" -#: ../lib/r10k/source/git.rb:77 +#: ../lib/r10k/source/git.rb:75 msgid "Fetching '%{remote}' to determine current branches." msgstr "" -#: ../lib/r10k/source/git.rb:80 +#: ../lib/r10k/source/git.rb:78 msgid "Unable to determine current branches for Git source '%{name}' (%{basedir})" msgstr "" -#: ../lib/r10k/source/git.rb:105 +#: ../lib/r10k/source/git.rb:113 msgid "Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}" msgstr "" -#: ../lib/r10k/source/git.rb:109 +#: ../lib/r10k/source/git.rb:122 msgid "Environment %{env_name} contained non-word characters, ignoring it." msgstr "" -#: ../lib/r10k/source/git.rb:128 ../lib/r10k/source/svn.rb:113 +#: ../lib/r10k/source/git.rb:141 ../lib/r10k/source/svn.rb:115 msgid "Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}" msgstr "" -#: ../lib/r10k/source/git.rb:139 +#: ../lib/r10k/source/git.rb:152 msgid "Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}" msgstr "" @@ -486,6 +574,10 @@ msgid "Both username and password must be specified" msgstr "" +#: ../lib/r10k/tarball.rb:167 +msgid "Cache not present at %{path}" +msgstr "" + #: ../lib/r10k/util/basedir.rb:34 msgid "Expected Array<#desired_contents>, got R10K::Deployment" msgstr "" @@ -506,41 +598,45 @@ msgid "pe_license feature is not available, PE only Puppet modules will not be downloadable." msgstr "" -#: ../lib/r10k/util/purgeable.rb:52 -msgid "Not purging %{item} due to internal exclusion match: %{exclusion_match}" +#: ../lib/r10k/util/purgeable.rb:91 +msgid "Not purging %{path} due to internal exclusion match: %{exclusion_match}" msgstr "" -#: ../lib/r10k/util/purgeable.rb:54 -msgid "Not purging %{item} due to whitelist match: %{whitelist_match}" +#: ../lib/r10k/util/purgeable.rb:93 +msgid "Not purging %{path} due to whitelist match: %{allowlist_match}" msgstr "" -#: ../lib/r10k/util/purgeable.rb:71 +#: ../lib/r10k/util/purgeable.rb:137 msgid "No unmanaged contents in %{managed_dirs}, nothing to purge" msgstr "" -#: ../lib/r10k/util/purgeable.rb:76 +#: ../lib/r10k/util/purgeable.rb:142 msgid "Removing unmanaged path %{path}" msgstr "" -#: ../lib/r10k/util/purgeable.rb:81 +#: ../lib/r10k/util/purgeable.rb:147 msgid "Unable to remove unmanaged path: %{path}" msgstr "" -#: ../lib/r10k/util/setopts.rb:49 +#: ../lib/r10k/util/setopts.rb:60 +msgid "%{class_name} parameters '%{a}' and '%{b}' conflict. Specify one or the other, but not both" +msgstr "" + +#: ../lib/r10k/util/setopts.rb:67 msgid "%{class_name} cannot handle option '%{key}'" msgstr "" -#: ../lib/r10k/util/subprocess.rb:69 +#: ../lib/r10k/util/subprocess.rb:70 msgid "Starting process: %{args}" msgstr "" -#: ../lib/r10k/util/subprocess.rb:74 +#: ../lib/r10k/util/subprocess.rb:75 msgid "" "Finished process:\n" "%{result}" msgstr "" -#: ../lib/r10k/util/subprocess.rb:77 +#: ../lib/r10k/util/subprocess.rb:78 msgid "Command exited with non-zero exit code" msgstr "" diff -Nru r10k-3.7.0/r10k.gemspec r10k-4.0.0/r10k.gemspec --- r10k-3.7.0/r10k.gemspec 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/r10k.gemspec 2023-08-01 15:06:51.000000000 +0000 @@ -18,32 +18,30 @@ dynamic environments. DESCRIPTION - s.required_ruby_version = '>= 2.3.0' + s.required_ruby_version = '>= 2.6.0' s.license = 'Apache-2.0' s.add_dependency 'colored2', '3.1.2' - s.add_dependency 'cri', ['>= 2.15.10', '< 3.0.0'] + s.add_dependency 'cri', '>= 2.15.10' s.add_dependency 'log4r', '1.1.10' s.add_dependency 'multi_json', '~> 1.10' - s.add_dependency 'puppet_forge', '~> 2.3.0' + s.add_dependency 'puppet_forge', '>= 4.1', '< 6' - s.add_dependency 'gettext-setup', '~>0.24' - # These two pins narrow what is allowed by gettext-setup, - # to preserver compatability with Ruby 2.4 - s.add_dependency 'fast_gettext', '~> 1.1.0' - s.add_dependency 'gettext', ['>= 3.0.2', '< 3.3.0'] + s.add_dependency 'gettext-setup', '>=0.24', '<2.0' + + s.add_dependency 'jwt', '>= 2.2.3', '< 2.8.0' + s.add_dependency 'minitar', '~> 0.9' s.add_development_dependency 'rspec', '~> 3.1' s.add_development_dependency 'rake' s.add_development_dependency 'yard', '~> 0.9.11' - s.add_development_dependency 'minitar', '~> 0.9.0' - s.files = %x[git ls-files].split($/) + s.files = %x[git ls-files].split($/).reject { |f| f.match(%r{^spec}) } s.require_path = 'lib' s.bindir = 'bin' s.executables = 'r10k' diff -Nru r10k-3.7.0/r10k.yaml.example r10k-4.0.0/r10k.yaml.example --- r10k-3.7.0/r10k.yaml.example 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/r10k.yaml.example 2023-08-01 15:06:51.000000000 +0000 @@ -110,3 +110,31 @@ # The 'baseurl' setting indicates where Forge modules should be installed # from. This defaults to 'https://forgeapi.puppetlabs.com' #baseurl: 'https://forgemirror.example.com' + +# Configuration options on how R10k should log its actions +logging: + # The 'level' setting sets the default log level to run R10k actions at. + # This value will be overridden by any value set through the command line. + #level: warn + + # Specify additional log outputs here, any log4r outputter can be used. + # If no log level is specified then the output will use the global level. + #outputs: + # - type: file + # level: debug + # parameters: + # filename: /var/log/r10k.log + # trunc: true + # - type: syslog + # - type: email + # only_at: [fatal] + # parameters: + # from: r10k@example.com + # to: sysadmins@example.com + # server: smtp.example.com + # subject: Fatal R10k error occurred + + # The 'disable_default_stderr' setting specifies if the default output on + # stderr should be active or not, in case R10k is to be run entirely + # through scripts or cronjobs where console output is unwelcome. + #disable_default_stderr: false Binary files /tmp/tmpody8kaze/JFXRpTjN2q/r10k-3.7.0/spec/fixtures/tarball/tarball.tar.gz and /tmp/tmpody8kaze/BJ5iiiBpQ6/r10k-4.0.0/spec/fixtures/tarball/tarball.tar.gz differ diff -Nru r10k-3.7.0/spec/fixtures/unit/action/r10k_creds.yaml r10k-4.0.0/spec/fixtures/unit/action/r10k_creds.yaml --- r10k-3.7.0/spec/fixtures/unit/action/r10k_creds.yaml 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/action/r10k_creds.yaml 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,9 @@ +--- + git: + private_key: '/global/config/private/key' + oauth_token: '/global/config/oauth/token' + repositories: + - remote: 'git@myfakegitserver.com:user/repo.git' + private_key: '/config/private/key' + - remote: 'https://myfakegitserver.com/user/repo.git' + oauth_token: '/config/oauth/token' diff -Nru r10k-3.7.0/spec/fixtures/unit/action/r10k_forge_auth.yaml r10k-4.0.0/spec/fixtures/unit/action/r10k_forge_auth.yaml --- r10k-3.7.0/spec/fixtures/unit/action/r10k_forge_auth.yaml 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/action/r10k_forge_auth.yaml 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,4 @@ +--- +forge: + baseurl: 'http://private-forge.com' + authorization_token: 'faketoken' diff -Nru r10k-3.7.0/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml r10k-4.0.0/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml --- r10k-3.7.0/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,3 @@ +--- +forge: + authorization_token: 'faketoken' diff -Nru r10k-3.7.0/spec/fixtures/unit/action/r10k_logging.yaml r10k-4.0.0/spec/fixtures/unit/action/r10k_logging.yaml --- r10k-3.7.0/spec/fixtures/unit/action/r10k_logging.yaml 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/action/r10k_logging.yaml 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,12 @@ +--- + logging: + level: FATAL + + outputs: + - type: file + parameters: + filename: r10k.log + + - type: syslog + + disable_default_stderr: true diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/forge-override/Puppetfile r10k-4.0.0/spec/fixtures/unit/puppetfile/forge-override/Puppetfile --- r10k-3.7.0/spec/fixtures/unit/puppetfile/forge-override/Puppetfile 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/forge-override/Puppetfile 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,8 @@ +forge "my.custom.forge.com" + +mod "puppetlabs/stdlib", '4.12.0' +mod "puppetlabs/concat", '2.1.0' + +mod 'apache', + :git => 'https://github.com/puppetlabs/puppetlabs-apache', + :branch => 'docs_experiment' diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,10 @@ +mod 'puppetlabs/apt', '2.1.1' +mod 'puppetlabs/stdlib', :latest +mod 'puppetlabs/concat' +mod 'puppetlabs/rpm', '2.1.1-pre1' +mod 'foo', git: 'this/remote', branch: 'main' +mod 'bar', git: 'this/remote', tag: 'v1.2.3' +mod 'baz', git: 'this/remote', commit: '123abc456' +mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678' +mod 'buzz', git: 'this/remote', ref: 'refs/heads/main' +mod 'canary', local: true diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/Puppetfile.new 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,10 @@ +mod 'puppetlabs/apt', '3.0.0' +mod 'puppetlabs/stdlib', :latest +mod 'puppetlabs/concat' +mod 'puppetlabs/rpm', '2.1.1-pre1' +mod 'foo', git: 'this/remote', branch: 'main' +mod 'bar', git: 'this/remote', tag: 'v1.2.3' +mod 'baz', git: 'this/remote', commit: '123abc456' +mod 'fizz', git: 'this/remote', ref: '1234567890abcdef1234567890abcdef12345678' +mod 'buzz', git: 'this/remote', ref: 'refs/heads/main' +mod 'canary', local: true diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/apt/.gitkeep 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/baz/.gitkeep 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/buzz/.gitkeep 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/canary/.gitkeep 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/fizz/.gitkeep 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -Nru r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep --- r10k-3.7.0/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/fixtures/unit/puppetfile/various-modules/modules/rpm/.gitkeep 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1 @@ +This only exists so the directory can be committed to git for testing purposes. \ No newline at end of file diff -Nru r10k-3.7.0/spec/integration/git/rugged/cache_spec.rb r10k-4.0.0/spec/integration/git/rugged/cache_spec.rb --- r10k-3.7.0/spec/integration/git/rugged/cache_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/integration/git/rugged/cache_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,33 @@ +require 'spec_helper' +require 'r10k/git/rugged/cache' + +describe R10K::Git::Rugged::Cache, :if => R10K::Features.available?(:rugged) do + include_context 'Git integration' + + let(:dirname) { 'working-repo' } + let(:remote_name) { 'origin' } + + subject { described_class.new(remote) } + + context "syncing with the remote" do + before(:each) do + subject.reset! + end + + describe "with the correct configuration" do + it "is able to sync with the remote" do + subject.sync + expect(subject.synced?).to eq(true) + end + end + + describe "with a out of date cached remote" do + it "updates the cached remote configuration" do + subject.repo.update_remote('foo', remote_name) + expect(subject.repo.remotes[remote_name]).to eq('foo') + subject.sync + expect(subject.repo.remotes[remote_name]).to eq(remote) + end + end + end +end diff -Nru r10k-3.7.0/spec/integration/git/stateful_repository_spec.rb r10k-4.0.0/spec/integration/git/stateful_repository_spec.rb --- r10k-3.7.0/spec/integration/git/stateful_repository_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/integration/git/stateful_repository_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -83,6 +83,22 @@ end end + describe "when the workdir has spec dir modifications" do + before(:each) do + thinrepo.clone(remote, {:ref => ref}) + FileUtils.mkdir_p(File.join(thinrepo.path, 'spec')) + File.open(File.join(thinrepo.path, 'spec', 'file_spec.rb'), 'a') { |f| f.write('local modifications!') } + thinrepo.stage_files(['spec/file_spec.rb']) + end + it "is dirty with exclude_spec false" do + expect(subject.status(ref, false)).to eq :dirty + end + + it "is insync with exclude_spec true" do + expect(subject.status(ref, true)).to eq :insync + end + end + describe "if the right ref is checked out" do it "is insync" do thinrepo.clone(remote, {:ref => ref}) diff -Nru r10k-3.7.0/spec/integration/util/purageable_spec.rb r10k-4.0.0/spec/integration/util/purageable_spec.rb --- r10k-3.7.0/spec/integration/util/purageable_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/integration/util/purageable_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,41 @@ +require 'spec_helper' +require 'r10k/util/purgeable' +require 'r10k/util/cleaner' + +require 'tmpdir' + +RSpec.describe R10K::Util::Purgeable do + it 'purges only unmanaged files' do + Dir.mktmpdir do |envdir| + managed_directory = "#{envdir}/managed_one" + desired_contents = [ + "#{managed_directory}/expected_1", + "#{managed_directory}/managed_subdir_1", + "#{managed_directory}/managed_symlink_dir", + "#{managed_directory}/managed_subdir_1/subdir_expected_1", + "#{managed_directory}/managed_subdir_1/managed_symlink_file", + ] + + FileUtils.cp_r('spec/fixtures/unit/util/purgeable/managed_one/', + managed_directory) + + cleaner = R10K::Util::Cleaner.new([managed_directory], desired_contents) + + cleaner.purge!({ recurse: true, whitelist: ["**/subdir_allowlisted_2"] }) + + # Files present after purge + expect(File.exist?("#{managed_directory}/expected_1")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1")).to be true + expect(File.exist?("#{managed_directory}/managed_symlink_dir")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_expected_1")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1/managed_symlink_file")).to be true + expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_allowlisted_2")).to be true + + # Purged files + expect(File.exist?("#{managed_directory}/unmanaged_1")).to be false + expect(File.exist?("#{managed_directory}/managed_subdir_1/unmanaged_symlink_dir")).to be false + expect(File.exist?("#{managed_directory}/unmanaged_symlink_file")).to be false + expect(File.exist?("#{managed_directory}/managed_subdir_1/subdir_unmanaged_1")).to be false + end + end +end diff -Nru r10k-3.7.0/spec/r10k-mocks/mock_env.rb r10k-4.0.0/spec/r10k-mocks/mock_env.rb --- r10k-3.7.0/spec/r10k-mocks/mock_env.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/r10k-mocks/mock_env.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,6 +1,9 @@ require 'r10k/environment' +require 'r10k/util/purgeable' class R10K::Environment::Mock < R10K::Environment::Base + include R10K::Util::Purgeable + def sync "synced" end diff -Nru r10k-3.7.0/spec/r10k-mocks/mock_source.rb r10k-4.0.0/spec/r10k-mocks/mock_source.rb --- r10k-3.7.0/spec/r10k-mocks/mock_source.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/r10k-mocks/mock_source.rb 2023-08-01 15:06:51.000000000 +0000 @@ -5,9 +5,13 @@ R10K::Source.register(:mock, self) def environments - corrected_environment_names = @options[:environments].map do |env| - R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn') + if @_environments.nil? + corrected_environment_names = @options[:environments].map do |env| + R10K::Environment::Name.new(env, :prefix => @prefix, :invalid => 'correct_and_warn') + end + @_environments = corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname, { overrides: @options[:overrides] }) } end - corrected_environment_names.map { |env| R10K::Environment::Mock.new(env.name, @basedir, env.dirname) } + + @_environments end end diff -Nru r10k-3.7.0/spec/shared-contexts/tarball.rb r10k-4.0.0/spec/shared-contexts/tarball.rb --- r10k-3.7.0/spec/shared-contexts/tarball.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/shared-contexts/tarball.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,32 @@ +require 'tmpdir' +require 'fileutils' + +shared_context "Tarball" do + # Suggested subject: + # + # subject { described_class.new('fixture-tarball', fixture_tarball, checksum: fixture_checksum) } + # + let(:fixture_tarball) do + File.expand_path('spec/fixtures/tarball/tarball.tar.gz', PROJECT_ROOT) + end + + let(:fixture_checksum) { '292e692ad18faabd4f9b21037d51f0185e04b69f82c522a54af91fb5b88c2d3b' } + + # Use tmpdir for cached tarballs + let(:tmpdir) { Dir.mktmpdir } + + # `moduledir` and `cache_root` are available for examples to use in creating + # their subjects + let(:moduledir) { File.join(tmpdir, 'modules').tap { |path| Dir.mkdir(path) } } + let(:cache_root) { File.join(tmpdir, 'cache').tap { |path| Dir.mkdir(path) } } + + around(:each) do |example| + if subject.is_a?(R10K::Tarball) + subject.settings[:cache_root] = cache_root + elsif subject.respond_to?(:tarball) && subject.tarball.is_a?(R10K::Tarball) + subject.tarball.settings[:cache_root] = cache_root + end + example.run + FileUtils.remove_entry_secure(tmpdir) + end +end diff -Nru r10k-3.7.0/spec/shared-examples/git/working_repository.rb r10k-4.0.0/spec/shared-examples/git/working_repository.rb --- r10k-3.7.0/spec/shared-examples/git/working_repository.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/shared-examples/git/working_repository.rb 2023-08-01 15:06:51.000000000 +0000 @@ -194,11 +194,13 @@ context "with local changes" do before(:each) do File.open(File.join(subject.path, 'README.markdown'), 'a') { |f| f.write('local modifications!') } + File.open(File.join(subject.path, 'CHANGELOG'), 'a') { |f| f.write('local modifications to the changelog too') } end it "logs and reports worktree as dirty" do expect(subject.logger).to receive(:debug).with(/found local modifications in.*README\.markdown/i) - expect(subject.logger).to receive(:debug1) + expect(subject.logger).to receive(:debug).with(/found local modifications in.*CHANGELOG/i) + expect(subject.logger).to receive(:debug1).twice expect(subject.dirty?).to be true end diff -Nru r10k-3.7.0/spec/shared-examples/puppetfile-action.rb r10k-4.0.0/spec/shared-examples/puppetfile-action.rb --- r10k-3.7.0/spec/shared-examples/puppetfile-action.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/shared-examples/puppetfile-action.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,15 +3,15 @@ shared_examples_for "a puppetfile action" do describe "initializing" do it "accepts the :root option" do - described_class.new({root: "/some/nonexistent/path"}, []) + described_class.new({root: "/some/nonexistent/path"}, [], {}) end it "accepts the :puppetfile option" do - described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, []) + described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, [], {}) end it "accepts the :moduledir option" do - described_class.new({moduledir: "/some/nonexistent/path/modules"}, []) + described_class.new({moduledir: "/some/nonexistent/path/modules"}, [], {}) end end @@ -20,19 +20,19 @@ shared_examples_for "a puppetfile install action" do describe "initializing" do it "accepts the :root option" do - described_class.new({root: "/some/nonexistent/path"}, []) + described_class.new({root: "/some/nonexistent/path"}, [], {}) end it "accepts the :puppetfile option" do - described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, []) + described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, [], {}) end it "accepts the :moduledir option" do - described_class.new({moduledir: "/some/nonexistent/path/modules"}, []) + described_class.new({moduledir: "/some/nonexistent/path/modules"}, [], {}) end it "accepts the :force option" do - described_class.new({force: true}, []) + described_class.new({force: true}, [], {}) end end diff -Nru r10k-3.7.0/spec/spec_helper.rb r10k-4.0.0/spec/spec_helper.rb --- r10k-3.7.0/spec/spec_helper.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/spec_helper.rb 2023-08-01 15:06:51.000000000 +0000 @@ -19,6 +19,7 @@ Dir.glob(File.expand_path('spec/shared-examples/**/*.rb', PROJECT_ROOT)).each { |file| require file } require 'shared-contexts/git-fixtures' +require 'shared-contexts/tarball' require 'matchers/exit_with' require 'matchers/match_realpath' require 'r10k-mocks' diff -Nru r10k-3.7.0/spec/unit/action/deploy/display_spec.rb r10k-4.0.0/spec/unit/action/deploy/display_spec.rb --- r10k-3.7.0/spec/unit/action/deploy/display_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/action/deploy/display_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -5,27 +5,57 @@ describe R10K::Action::Deploy::Display do describe "initializing" do it "accepts a puppetfile option" do - described_class.new({puppetfile: true}, []) + described_class.new({puppetfile: true}, [], {}) + end + + it "accepts a modules option" do + described_class.new({modules: true}, [], {}) end it "accepts a detail option" do - described_class.new({detail: true}, []) + described_class.new({detail: true}, [], {}) end it "accepts a format option" do - described_class.new({format: "json"}, []) + described_class.new({format: "json"}, [], {}) end it "accepts a fetch option" do - described_class.new({fetch: true}, []) + described_class.new({fetch: true}, [], {}) end end - subject { described_class.new({config: "/some/nonexistent/path"}, []) } + subject { described_class.new({config: "/some/nonexistent/path"}, [], {}) } before do allow(subject).to receive(:puts) end it_behaves_like "a deploy action that requires a config file" + + describe "collecting info" do + subject { described_class.new({config: "/some/nonexistent/path", format: 'json', puppetfile: true, detail: true}, ['first'], {}) } + + let(:mock_config) do + R10K::Deployment::MockConfig.new( + :sources => { + :control => { + :type => :mock, + :basedir => '/some/nonexistent/path/control', + :environments => %w[first second third env-that/will-be-corrected], + :prefix => 'PREFIX' + } + } + ) + end + + let(:deployment) { R10K::Deployment.new(mock_config) } + + it "gathers environment info" do + source_info = subject.send(:source_info, deployment.sources.first, ['first']) + expect(source_info[:name]).to eq(:control) + expect(source_info[:environments].length).to eq(1) + expect(source_info[:environments][0][:name]).to eq('first') + end + end end diff -Nru r10k-3.7.0/spec/unit/action/deploy/environment_spec.rb r10k-4.0.0/spec/unit/action/deploy/environment_spec.rb --- r10k-3.7.0/spec/unit/action/deploy/environment_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/action/deploy/environment_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -5,36 +5,66 @@ describe R10K::Action::Deploy::Environment do - subject { described_class.new({config: "/some/nonexistent/path"}, []) } + subject { described_class.new({config: "/some/nonexistent/path"}, [], {}) } it_behaves_like "a deploy action that can be write locked" it_behaves_like "a deploy action that requires a config file" describe "initializing" do it "can accept a cachedir option" do - described_class.new({cachedir: "/some/nonexistent/cachedir"}, []) + described_class.new({cachedir: "/some/nonexistent/cachedir"}, [], {}) end it "can accept a puppetfile option" do - described_class.new({puppetfile: true}, []) + described_class.new({puppetfile: true}, [], {}) + end + + it "can accept a modules option" do + described_class.new({modules: true}, [], {}) end it "can accept a default_branch_override option" do - described_class.new({:'default-branch-override' => 'default_branch_override_name'}, []) + described_class.new({:'default-branch-override' => 'default_branch_override_name'}, [], {}) end it "can accept a no-force option" do - described_class.new({:'no-force' => true}, []) + described_class.new({:'no-force' => true}, [], {}) end - it "normalizes environment names in the arg vector" - it 'can accept a generate-types option' do - described_class.new({ 'generate-types': true }, []) + described_class.new({ 'generate-types': true }, [], {}) end it 'can accept a puppet-path option' do - described_class.new({ 'puppet-path': '/nonexistent' }, []) + described_class.new({ 'puppet-path': '/nonexistent' }, [], {}) + end + + it 'can accept a private-key option' do + described_class.new({ 'private-key': '/nonexistent' }, [], {}) + end + + it 'can accept a token option' do + described_class.new({ 'oauth-token': '/nonexistent' }, [], {}) + end + + it 'can accept an app id option' do + described_class.new({ 'github-app-id': '/nonexistent' }, [], {}) + end + + it 'can accept a ttl option' do + described_class.new({ 'github-app-ttl': '/nonexistent' }, [], {}) + end + + it 'can accept a ssl private key option' do + described_class.new({ 'github-app-key': '/nonexistent' }, [], {}) + end + + it 'can accept a exclude-spec option' do + described_class.new({ :'exclude-spec' => true }, [], {}) + end + + it 'can accept an incremental option' do + described_class.new({ :incremental => true }, [], {}) end end @@ -52,6 +82,72 @@ ) end + describe "with puppetfile or modules flag" do + let(:deployment) { R10K::Deployment.new(mock_config) } + let(:loader) do + instance_double("R10K::ModuleLoader::Puppetfile", + :load => { + :modules => ['foo'], + :purge_exclusions => [], + :managed_directories => [], + :desired_contents => [] + } + ).as_null_object + end + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + and_return(loader).at_least(:once) + end + + it "syncs the puppetfile content when given the puppetfile flag" do + expect(loader).to receive(:load).exactly(4).times + expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times + action = described_class.new({config: "/some/nonexistent/path", puppetfile: true}, [], {}) + action.call + end + + it "syncs the puppetfile when given the modules flag" do + expect(loader).to receive(:load).exactly(4).times + expect(R10K::ContentSynchronizer).to receive(:concurrent_sync).exactly(4).times + action = described_class.new({config: "/some/nonexistent/path", modules: true}, [], {}) + action.call + end + end + + describe "with incremental flag" do + let(:loader) do + instance_double("R10K::ModuleLoader::Puppetfile", + :load => { + :modules => ['foo'], + :purge_exclusions => [], + :managed_directories => [], + :desired_contents => [] + } + ).as_null_object + end + + before do + expect(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings| + original.call(mock_config.merge(settings)) + end + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + and_return(loader).at_least(:once) + end + + it "incremental flag causes the module definitons to be preloaded by the loader" do + expect(loader).to receive(:load_metadata).exactly(4).times + action = described_class.new({:config => "/some/nonexistent/path", + :modules => true, + :incremental => true}, + [], + {}) + action.call + end + end + + describe "with an environment that doesn't exist" do let(:deployment) do R10K::Deployment.new(mock_config) @@ -61,7 +157,7 @@ expect(R10K::Deployment).to receive(:new).and_return(deployment) end - subject { described_class.new({config: "/some/nonexistent/path"}, %w[not_an_environment]) } + subject { described_class.new({config: "/some/nonexistent/path"}, %w[not_an_environment], {}) } it "logs that the environments can't be deployed and returns false" do expect(subject.logger).to receive(:error).with("Environment(s) 'not_an_environment' cannot be found in any source and will not be deployed.") @@ -71,10 +167,10 @@ end describe "with no-force" do - subject { described_class.new({ config: "/some/nonexistent/path", puppetfile: true, :'no-force' => true}, %w[first]) } + subject { described_class.new({ config: "/some/nonexistent/path", modules: true, :'no-force' => true}, %w[first], {}) } it "tries to preserve local modifications" do - expect(subject.force).to equal(false) + expect(subject.settings[:overrides][:modules][:force]).to equal(false) end end @@ -162,26 +258,75 @@ end end + describe "Purging allowlist" do + + let(:settings) { { pool_size: 4, deploy: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } } + let(:overrides) { { environments: {}, modules: { pool_size: 4 }, purging: { purge_levels: [:environment], purge_allowlist: ['coolfile', 'coolfile2'] } } } + let(:deployment) do + R10K::Deployment.new(mock_config.merge({overrides: overrides})) + end + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + allow_any_instance_of(R10K::Environment::Base).to receive(:purge!) + end + + subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) } + + it "reads in the purge_allowlist setting and purges accordingly" do + expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/) + expect(subject.settings[:overrides][:purging][:purge_allowlist]).to eq(['coolfile', 'coolfile2']) + subject.call + end + end + describe "purge_levels" do let(:settings) { { deploy: { purge_levels: purge_levels } } } + let(:overrides) do + { + environments: { + requested_environments: ['PREFIX_first'] + }, + modules: { + deploy_modules: true, + pool_size: 4 + }, + purging: { + purge_levels: purge_levels + } + } + end let(:deployment) do - R10K::Deployment.new(mock_config.merge(settings)) + R10K::Deployment.new(mock_config.merge({ overrides: overrides })) end before do expect(R10K::Deployment).to receive(:new).and_return(deployment) + allow_any_instance_of(R10K::Environment::Base).to receive(:purge!) end - subject { described_class.new({ config: "/some/nonexistent/path", puppetfile: true }, %w[PREFIX_first], settings) } + subject { described_class.new({ config: "/some/nonexistent/path", modules: true }, %w[PREFIX_first], settings) } describe "deployment purge level" do let(:purge_levels) { [:deployment] } + + it "updates the source's cache before it purges environments" do + deployment.sources.each do |source| + expect(source).to receive(:reload!).ordered + end + expect(deployment).to receive(:purge!).ordered + subject.call + end + it "only logs about purging deployment" do - expect(subject.logger).to receive(:debug).with(/purging unmanaged environments for deployment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i) + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/) + original.call(env) + end.at_least(:once) + + expect(subject.logger).to receive(:debug).with(/Purging unmanaged environments for deployment/) + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/) subject.call end @@ -191,17 +336,25 @@ let(:purge_levels) { [:environment] } it "only logs about purging environment" do - expect(subject.logger).to receive(:debug).with(/purging unmanaged content for environment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged puppetfile content/i) + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + expect(env.logger).to_not receive(:debug).with(/Purging unmanaged puppetfile content/) + original.call(env) + end.at_least(:once) + expect(subject.logger).to receive(:debug).with(/Purging unmanaged content for environment/) + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/) subject.call end it "logs that environment was not purged if deploy failed" do - expect(subject).to receive(:visit_puppetfile) { subject.instance_variable_set(:@visit_ok, false) } + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + if env.name =~ /first/ + expect(env).to receive(:deploy) { subject.instance_variable_set(:@visit_ok, false) } + end + original.call(env) + end.at_least(:once) - expect(subject.logger).to receive(:debug).with(/not purging unmanaged content for environment/i) + expect(subject.logger).to receive(:debug).with(/Not purging unmanaged content for environment/) subject.call end @@ -211,14 +364,22 @@ let(:purge_levels) { [:puppetfile] } it "only logs about purging puppetfile" do - expect(subject.logger).to receive(:debug).with(/purging unmanaged puppetfile content/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged environments for deployment/i) - expect(subject.logger).to_not receive(:debug).with(/purging unmanaged content for environment/i) + allow(R10K::ContentSynchronizer).to receive(:concurrent_sync) + expect(subject).to receive(:visit_environment).and_wrap_original do |original, env, &block| + if env.name =~ /first/ + expect(env.logger).to receive(:debug).with(/Purging unmanaged Puppetfile content/) + end + original.call(env) + end.at_least(:once) + + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged environments for deployment/) + expect(subject.logger).to_not receive(:debug).with(/Purging unmanaged content for environment/) subject.call end end end + describe "generate-types" do let(:deployment) do R10K::Deployment.new( @@ -229,6 +390,11 @@ basedir: '/some/nonexistent/path/control', environments: %w[first second] } + }, + overrides: { + modules: { + pool_size: 4 + } } ) ) @@ -236,9 +402,8 @@ before do allow(R10K::Deployment).to receive(:new).and_return(deployment) - end + allow_any_instance_of(R10K::Environment::Base).to receive(:purge!) - before(:each) do allow(subject).to receive(:write_environment_info!) expect(subject.logger).not_to receive(:error) end @@ -248,19 +413,22 @@ described_class.new( { config: '/some/nonexistent/path', - puppetfile: true, + modules: true, 'generate-types': true }, - %w[first second] + %w[first second], + {} ) end it 'generate_types is true' do - expect(subject.instance_variable_get(:@generate_types)).to eq(true) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(true) end it 'only calls puppet generate types on specified environment' do - subject.instance_variable_set(:@argv, %w[first]) + settings = subject.instance_variable_get(:@settings) + settings[:overrides][:environments][:requested_environments] = %w{first} + subject.instance_variable_set(:@settings, settings) expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| if environment.dirname == 'first' expect(environment).to receive(:generate_types!) @@ -273,8 +441,8 @@ end it 'does not call puppet generate types on puppetfile failure' do - allow(subject).to receive(:visit_puppetfile) { subject.instance_variable_set(:@visit_ok, false) } expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + allow(environment).to receive(:deploy) { subject.instance_variable_set(:@visit_ok, false) } expect(environment).not_to receive(:generate_types!) original.call(environment, &block) end.twice @@ -282,10 +450,11 @@ end it 'calls puppet generate types on previous puppetfile failure' do - allow(subject).to receive(:visit_puppetfile) do |puppetfile| - subject.instance_variable_set(:@visit_ok, false) if puppetfile.environment.dirname == 'first' - end expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + allow(environment).to receive(:deploy) do + subject.instance_variable_set(:@visit_ok, false) if environment.dirname == 'first' + end + if environment.dirname == 'second' expect(environment).to receive(:generate_types!) else @@ -302,15 +471,16 @@ described_class.new( { config: '/some/nonexistent/path', - puppetfile: true, + modules: true, 'generate-types': false }, - %w[first] + %w[first], + {} ) end it 'generate_types is false' do - expect(subject.instance_variable_get(:@generate_types)).to eq(false) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(false) end it 'does not call puppet generate types' do @@ -325,7 +495,7 @@ describe 'with puppet-path' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, [], {}) } it 'sets puppet_path' do expect(subject.instance_variable_get(:@puppet_path)).to eq('/nonexistent') @@ -334,12 +504,30 @@ describe 'with puppet-conf' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, [], {}) } it 'sets puppet_conf' do expect(subject.instance_variable_get(:@puppet_conf)).to eq('/nonexistent') end end + + describe 'with private-key' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'private-key': '/nonexistent' }, [], {}) } + + it 'sets private_key' do + expect(subject.instance_variable_get(:@private_key)).to eq('/nonexistent') + end + end + + describe 'with oauth-token' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'oauth-token': '/nonexistent' }, [], {}) } + + it 'sets oauth_token' do + expect(subject.instance_variable_get(:@oauth_token)).to eq('/nonexistent') + end + end end describe "write_environment_info!" do @@ -352,20 +540,35 @@ def initialize(path, info) @path = path @info = info - @puppetfile = R10K::Puppetfile.new + @puppetfile = R10K::Puppetfile.new("", {}) end end let(:mock_stateful_repo_1) { instance_double("R10K::Git::StatefulRepository", :head => "123456") } let(:mock_stateful_repo_2) { instance_double("R10K::Git::StatefulRepository", :head => "654321") } - let(:mock_git_module_1) { instance_double("R10K::Module::Git", :name => "my_cool_module", :version => "1.0", :repo => mock_stateful_repo_1) } - let(:mock_git_module_2) { instance_double("R10K::Module::Git", :name => "my_lame_module", :version => "0.0.1", :repo => mock_stateful_repo_2) } - let(:mock_forge_module_1) { double(:name => "their_shiny_module", :version => "2.0.0") } - let(:mock_puppetfile) { instance_double("R10K::Puppetfile", :modules => [mock_git_module_1, mock_git_module_2, mock_forge_module_1]) } + let(:mock_git_module_1) do + instance_double("R10K::Module::Git", + :name => "my_cool_module", + :properties => { + :type => :git, + :expected => "1.0", + :actual => mock_stateful_repo_1.head + }) + end + let(:mock_git_module_2) do + instance_double("R10K::Module::Git", + :name => "my_uncool_module", + :properties => { + :type => :git, + :expected => "0.0.1", + :actual => mock_stateful_repo_2.head + }) + end + let(:mock_forge_module_1) { double(:name => "their_shiny_module", :properties => { :expected => "2.0.0" }) } before(:all) do @tmp_path = "./tmp-r10k-test-dir/" - Dir.mkdir(@tmp_path) unless File.exists?(@tmp_path) + Dir.mkdir(@tmp_path) unless File.exist?(@tmp_path) end after(:all) do @@ -373,12 +576,9 @@ Dir.delete(@tmp_path) end - it "writes the .r10k-deploy file correctly" do - allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile) - allow(mock_forge_module_1).to receive(:repo).and_raise(NoMethodError) - + it "writes the .r10k-deploy file correctly if all goes well" do fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"}) - allow(fake_env).to receive(:modules).and_return(mock_puppetfile.modules) + allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1]) subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true) file_contents = File.read("#{@tmp_path}/.r10k-deploy.json") @@ -392,13 +592,28 @@ expect(r10k_deploy['module_deploys'][0]['name']).to eq("my_cool_module") expect(r10k_deploy['module_deploys'][0]['version']).to eq("1.0") expect(r10k_deploy['module_deploys'][0]['sha']).to eq("123456") - expect(r10k_deploy['module_deploys'][1]['name']).to eq("my_lame_module") + expect(r10k_deploy['module_deploys'][1]['name']).to eq("my_uncool_module") expect(r10k_deploy['module_deploys'][1]['version']).to eq("0.0.1") expect(r10k_deploy['module_deploys'][1]['sha']).to eq("654321") expect(r10k_deploy['module_deploys'][2]['name']).to eq("their_shiny_module") expect(r10k_deploy['module_deploys'][2]['version']).to eq("2.0.0") expect(r10k_deploy['module_deploys'][2]['sha']).to eq(nil) + end + + it "writes the .r10k-deploy file correctly if there's a failure" do + fake_env = Fake_Environment.new(@tmp_path, {:name => "my_cool_environment", :signature => "pablo picasso"}) + allow(fake_env).to receive(:modules).and_return([mock_git_module_1, mock_git_module_2, mock_forge_module_1]) + allow(mock_forge_module_1).to receive(:properties).and_raise(StandardError) + subject.send(:write_environment_info!, fake_env, "2019-01-01 23:23:22 +0000", true) + file_contents = File.read("#{@tmp_path}/.r10k-deploy.json") + r10k_deploy = JSON.parse(file_contents) + + expect(r10k_deploy['name']).to eq("my_cool_environment") + expect(r10k_deploy['signature']).to eq("pablo picasso") + expect(r10k_deploy['started_at']).to eq("2019-01-01 23:23:22 +0000") + expect(r10k_deploy['deploy_success']).to eq(true) + expect(r10k_deploy['module_deploys'].length).to eq(0) end end end diff -Nru r10k-3.7.0/spec/unit/action/deploy/module_spec.rb r10k-4.0.0/spec/unit/action/deploy/module_spec.rb --- r10k-3.7.0/spec/unit/action/deploy/module_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/action/deploy/module_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -4,43 +4,67 @@ describe R10K::Action::Deploy::Module do - subject { described_class.new({config: "/some/nonexistent/path"}, []) } + subject { described_class.new({config: "/some/nonexistent/path"}, [], {}) } it_behaves_like "a deploy action that requires a config file" it_behaves_like "a deploy action that can be write locked" describe "initializing" do it "accepts an environment option" do - described_class.new({environment: "production"}, []) + described_class.new({environment: "production"}, [], {}) end it "can accept a no-force option" do - described_class.new({:'no-force' => true}, []) + described_class.new({:'no-force' => true}, [], {}) end it 'can accept a generate-types option' do - described_class.new({ 'generate-types': true }, []) + described_class.new({ 'generate-types': true }, [], {}) end it 'can accept a puppet-path option' do - described_class.new({ 'puppet-path': '/nonexistent' }, []) + described_class.new({ 'puppet-path': '/nonexistent' }, [], {}) end it 'can accept a puppet-conf option' do - described_class.new({ 'puppet-conf': '/nonexistent' }, []) + described_class.new({ 'puppet-conf': '/nonexistent' }, [], {}) end it 'can accept a cachedir option' do - described_class.new({ cachedir: '/nonexistent' }, []) + described_class.new({ cachedir: '/nonexistent' }, [], {}) + end + + it 'can accept a private-key option' do + described_class.new({ 'private-key': '/nonexistent' }, [], {}) + end + + it 'can accept a token option' do + described_class.new({ 'oauth-token': '/nonexistent' }, [], {}) + end + + it 'can accept an app id option' do + described_class.new({ 'github-app-id': '/nonexistent' }, [], {}) + end + + it 'can accept a ttl option' do + described_class.new({ 'github-app-ttl': '/nonexistent' }, [], {}) + end + + it 'can accept a ssl private key option' do + described_class.new({ 'github-app-key': '/nonexistent' }, [], {}) + end + + it 'can accept a exclude-spec option' do + described_class.new({ :'exclude-spec' => true }, [], {}) end end describe "with no-force" do - subject { described_class.new({ config: "/some/nonexistent/path", :'no-force' => true}, [] )} + subject { described_class.new({ config: "/some/nonexistent/path", :'no-force' => true}, [], {}) } it "tries to preserve local modifications" do - expect(subject.force).to equal(false) + expect(subject.settings[:overrides][:modules][:force]).to equal(false) end end @@ -53,6 +77,11 @@ basedir: '/some/nonexistent/path/control', environments: %w[first second] } + }, + overrides: { + modules: { + pool_size: 4 + } } ) end @@ -68,32 +97,26 @@ config: '/some/nonexistent/path', 'generate-types': true }, - %w[first] + %w[first], + {} ) end - before do - allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| - expect(environment.puppetfile).to receive(:modules_by_vcs_cachedir).and_return( - {none: [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]} - ) - original.call(environment, &block) - end - end - it 'generate_types is true' do - expect(subject.instance_variable_get(:@generate_types)).to eq(true) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(true) end - it 'only calls puppet generate types on environments with specified module' do - expect(subject).to receive(:visit_module).and_wrap_original do |original, mod, &block| - if mod.name == 'first' - expect(mod.environment).to receive(:generate_types!) + it 'only calls puppet generate types on environments where the specified module was updated' do + allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + if environment.name == 'first' + expect(environment).to receive(:deploy).and_return(['first']) + expect(environment).to receive(:generate_types!) else - expect(mod.environment).not_to receive(:generate_types!) + expect(environment).to receive(:deploy).and_return([]) + expect(environment).not_to receive(:generate_types!) end - original.call(mod, &block) - end.twice + original.call(environment, &block) + end subject.call end end @@ -105,12 +128,13 @@ config: '/some/nonexistent/path', 'generate-types': false }, - %w[first] + %w[first], + {} ) end it 'generate_types is false' do - expect(subject.instance_variable_get(:@generate_types)).to eq(false) + expect(subject.settings[:overrides][:environments][:generate_types]).to eq(false) end it 'does not call puppet generate types' do |it| @@ -125,7 +149,7 @@ describe 'with puppet-path' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-path': '/nonexistent' }, [], {}) } it 'sets puppet_path' do expect(subject.instance_variable_get(:@puppet_path)).to eq('/nonexistent') @@ -134,7 +158,7 @@ describe 'with puppet-conf' do - subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, [], {}) } it 'sets puppet_conf' do expect(subject.instance_variable_get(:@puppet_conf)).to eq('/nonexistent') @@ -143,10 +167,310 @@ describe 'with cachedir' do - subject { described_class.new({ config: '/some/nonexistent/path', cachedir: '/nonexistent' }, []) } + subject { described_class.new({ config: '/some/nonexistent/path', cachedir: '/nonexistent' }, [], {}) } - it 'sets puppet_path' do + it 'sets cachedir' do expect(subject.instance_variable_get(:@cachedir)).to eq('/nonexistent') end end + + describe 'with private-key' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'private-key': '/nonexistent' }, [], {}) } + + it 'sets private_key' do + expect(subject.instance_variable_get(:@private_key)).to eq('/nonexistent') + end + end + + describe 'with oauth-token' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'oauth-token': '/nonexistent' }, [], {}) } + + it 'sets token_path' do + expect(subject.instance_variable_get(:@oauth_token)).to eq('/nonexistent') + end + end + + describe 'with github-app-id' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'github-app-id': '/nonexistent' }, [], {}) } + + it 'sets github-app-id' do + expect(subject.instance_variable_get(:@github_app_id)).to eq('/nonexistent') + end + end + + describe 'with github-app-key' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'github-app-key': '/nonexistent' }, [], {}) } + + it 'sets github-app-key' do + expect(subject.instance_variable_get(:@github_app_key)).to eq('/nonexistent') + end + end + + describe 'with github-app-ttl' do + + subject { described_class.new({ config: '/some/nonexistent/path', 'github-app-ttl': '/nonexistent' }, [], {}) } + + it 'sets github-app-ttl' do + expect(subject.instance_variable_get(:@github_app_ttl)).to eq('/nonexistent') + end + end + + describe 'with modules' do + + subject { described_class.new({ config: '/some/nonexistent/path' }, ['mod1', 'mod2'], {}) } + + let(:cache) { instance_double("R10K::Git::Cache", 'sanitized_dirname' => 'foo', 'cached?' => true, 'sync' => true) } + let(:repo) { instance_double("R10K::Git::StatefulRepository", cache: cache, resolve: 'main', tracked_paths: []) } + + it 'does not sync modules not given' do + allow(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings, &block| + original.call(settings.merge({ + sources: { + main: { + remote: 'https://not/a/remote', + basedir: '/not/a/basedir', + type: 'git' + } + } + })) + end + + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + allow(R10K::Git).to receive_message_chain(:cache, :generate).and_return(cache) + allow_any_instance_of(R10K::Source::Git).to receive(:environment_names).and_return([R10K::Environment::Name.new('first', {})]) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + # For this test we want to have realistic Modules and access to + # their internal Repos to validate the sync. Unfortunately, to + # do so we do some invasive mocking, effectively implementing + # our own R10K::ModuleLoader::Puppetfile#load. We directly update + # the Environment's internal ModuleLoader and then call `load` on + # it so it will create the correct loaded_content. + loader = environment.loader + allow(loader).to receive(:puppetfile_content).and_return('') + expect(loader).to receive(:load) do + loader.add_module('mod1', { git: 'https://remote', default_branch: 'main'}) + loader.add_module('mod2', { git: 'https://remote', default_branch: 'main'}) + loader.add_module('mod3', { git: 'https://remote', default_branch: 'main'}) + + loaded_content = loader.load! + loaded_content[:modules].each do |mod| + if ['mod1', 'mod2'].include?(mod.name) + expect(mod.should_sync?).to be(true) + else + expect(mod.should_sync?).to be(false) + end + expect(mod).to receive(:sync).and_call_original + end + + loaded_content + end + + original.call(environment, &block) + end + + expect(repo).to receive(:sync).twice + + subject.call + end + end + + describe 'with environments' do + subject { described_class.new({ config: '/some/nonexistent/path', environment: 'first' }, ['mod1'], {}) } + + let(:cache) { instance_double("R10K::Git::Cache", 'sanitized_dirname' => 'foo', 'cached?' => true, 'sync' => true) } + let(:repo) { instance_double("R10K::Git::StatefulRepository", cache: cache, resolve: 'main', tracked_paths: []) } + + it 'only syncs to the given environments' do + allow(R10K::Deployment).to receive(:new).and_wrap_original do |original, settings, &block| + original.call(settings.merge({ + sources: { + main: { + remote: 'https://not/a/remote', + basedir: '/not/a/basedir', + type: 'git' + } + } + })) + end + + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + allow(R10K::Git).to receive_message_chain(:cache, :generate).and_return(cache) + allow_any_instance_of(R10K::Source::Git).to receive(:environment_names).and_return([R10K::Environment::Name.new('first', {}), + R10K::Environment::Name.new('second', {})]) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + loader = environment.loader + + if environment.name == 'first' + # For this test we want to have realistic Modules and access to + # their internal Repos to validate the sync. Unfortunately, to + # do so we do some invasive mocking, effectively implementing + # our own R10K::ModuleLoader::Puppetfile#load. We directly update + # the Environment's internal ModuleLoader and then call `load` on + # it so it will create the correct loaded_content. + allow(loader).to receive(:puppetfile_content).and_return('') + expect(loader).to receive(:load) do + loader.add_module('mod1', { git: 'https://remote', default_branch: 'main'}) + loader.add_module('mod2', { git: 'https://remote', default_branch: 'main'}) + + loaded_content = loader.load! + loaded_content[:modules].each do |mod| + if mod.name == 'mod1' + expect(mod.should_sync?).to be(true) + else + expect(mod.should_sync?).to be(false) + end + expect(mod).to receive(:sync).and_call_original + end + + loaded_content + end + + else + expect(loader).not_to receive(:load) + end + + original.call(environment, &block) + end.twice + + expect(repo).to receive(:sync).once + expect(subject.logger).to receive(:debug1).with(/Updating modules.*in environment.*first/i) + expect(subject.logger).to receive(:debug1).with(/skipping environment.*second/i) + + subject.call + end + end + + + describe "postrun" do + let(:mock_config) do + R10K::Deployment::MockConfig.new( + :sources => { + :control => { + :type => :mock, + :basedir => '/some/nonexistent/path/control', + :environments => %w[first second third], + } + } + ) + end + + context "basic postrun hook" do + let(:settings) { { postrun: ["/path/to/executable", "arg1", "arg2"] } } + let(:deployment) { R10K::Deployment.new(mock_config.merge(settings)) } + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + end + + subject do + described_class.new({config: "/some/nonexistent/path" }, + ['mod1'], settings) + end + + it "is passed to Subprocess" do + mock_subprocess = double + allow(mock_subprocess).to receive(:logger=) + expect(mock_subprocess).to receive(:execute) + + expect(R10K::Util::Subprocess).to receive(:new). + with(["/path/to/executable", "arg1", "arg2"]). + and_return(mock_subprocess) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + modified = subject.instance_variable_get(:@modified_envs) << environment + subject.instance_variable_set(:modified_envs, modified) + end.exactly(3).times + + subject.call + end + end + + context "supports environments" do + context "with one environment" do + let(:settings) { { postrun: ["/generate/types/wrapper", "$modifiedenvs"] } } + let(:deployment) { R10K::Deployment.new(mock_config.merge(settings)) } + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + end + + subject do + described_class.new({ config: '/some/nonexistent/path', + environment: 'first' }, + ['mod1'], settings) + end + + it "properly substitutes the environment" do + mock_subprocess = double + allow(mock_subprocess).to receive(:logger=) + expect(mock_subprocess).to receive(:execute) + + expect(R10K::Util::Subprocess).to receive(:new). + with(["/generate/types/wrapper", "first"]). + and_return(mock_subprocess) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + if environment.name == 'first' + expect(environment).to receive(:deploy).and_return(['first']) + end + original.call(environment, &block) + end.exactly(3).times + + subject.call + end + end + + context "with all environments" do + let(:settings) { { postrun: ["/generate/types/wrapper", "$modifiedenvs"] } } + let(:deployment) { R10K::Deployment.new(mock_config.merge(settings)) } + + before do + expect(R10K::Deployment).to receive(:new).and_return(deployment) + end + + subject do + described_class.new({ config: '/some/nonexistent/path' }, + ['mod1'], settings) + end + + it "properly substitutes the environment where modules were deployed" do + mock_subprocess = double + allow(mock_subprocess).to receive(:logger=) + expect(mock_subprocess).to receive(:execute) + + expect(R10K::Util::Subprocess).to receive(:new). + with(["/generate/types/wrapper", "first third"]). + and_return(mock_subprocess) + + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + if ['first', 'third'].include?(environment.name) + expect(environment).to receive(:deploy).and_return(['mod1']) + end + original.call(environment, &block) + end.exactly(3).times + + subject.call + end + + it "does not execute the command if no envs had the module" do + expect(R10K::Util::Subprocess).not_to receive(:new) + + mock_mod2 = double('mock_mod', name: 'mod2') + expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block| + expect(environment).to receive(:deploy).and_return([]) + original.call(environment, &block) + end.exactly(3).times + + subject.call + end + end + end + end end + diff -Nru r10k-3.7.0/spec/unit/action/puppetfile/check_spec.rb r10k-4.0.0/spec/unit/action/puppetfile/check_spec.rb --- r10k-3.7.0/spec/unit/action/puppetfile/check_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/action/puppetfile/check_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,7 +3,14 @@ describe R10K::Action::Puppetfile::Check do let(:default_opts) { {root: "/some/nonexistent/path"} } - let(:puppetfile) { instance_double('R10K::Puppetfile', :load! => true) } + let(:modules) do + [R10K::Module::Git.new("author/modname", + "/some/nonexistent/path/modname", + {git: 'https://my/git/remote', branch: 'main'})] + end + + + let(:loader) { instance_double('R10K::ModuleLoader::Puppetfile', :load! => {}, :modules => modules) } def checker(opts = {}, argv = [], settings = {}) opts = default_opts.merge(opts) @@ -11,11 +18,43 @@ end before(:each) do - allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil).and_return(puppetfile) + allow(R10K::ModuleLoader::Puppetfile). + to receive(:new). + with({ + basedir: "/some/nonexistent/path", + overrides: {modules: {default_ref: nil}} + }).and_return(loader) end it_behaves_like "a puppetfile action" + describe 'when no ref is defined' do + let(:modules) do + [R10K::Module::Git.new("author/modname", + "/some/nonexistent/path/modname", + {git: 'https://my/git/remote'})] + end + + it 'returns an error message' do + expect($stderr).to receive(:puts).with(/no ref defined/i) + checker.call + end + end + + describe 'when a default_ref is defined' do + let(:modules) do + [R10K::Module::Git.new("author/modname", + "/some/nonexistent/path/modname", + {git: 'https://my/git/remote', + overrides: {modules: {default_ref: 'main'}}})] + end + + it 'is valid syntax' do + expect($stderr).to receive(:puts).with(/Syntax OK/i) + checker.call + end + end + it "prints 'Syntax OK' when the Puppetfile syntax could be validated" do expect($stderr).to receive(:puts).with("Syntax OK") @@ -23,8 +62,11 @@ end it "prints an error message when validating the Puppetfile syntax raised an error" do - allow(puppetfile).to receive(:load!).and_raise(R10K::Error.new("Boom!")) - allow(R10K::Errors::Formatting).to receive(:format_exception).with(instance_of(R10K::Error), anything).and_return("Formatted error message") + allow(loader).to receive(:load!).and_raise(R10K::Error.new("Boom!")) + allow(R10K::Errors::Formatting). + to receive(:format_exception). + with(instance_of(R10K::Error), anything). + and_return("Formatted error message") expect($stderr).to receive(:puts).with("Formatted error message") @@ -34,7 +76,13 @@ it "respects --puppetfile option" do allow($stderr).to receive(:puts) - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/custom/puppetfile/path").and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile). + to receive(:new). + with({ + basedir: "/some/nonexistent/path", + overrides: {modules: {default_ref: nil}}, + puppetfile: "/custom/puppetfile/path" + }).and_return(loader) checker({puppetfile: "/custom/puppetfile/path"}).call end diff -Nru r10k-3.7.0/spec/unit/action/puppetfile/install_spec.rb r10k-4.0.0/spec/unit/action/puppetfile/install_spec.rb --- r10k-3.7.0/spec/unit/action/puppetfile/install_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/action/puppetfile/install_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -2,8 +2,12 @@ require 'r10k/action/puppetfile/install' describe R10K::Action::Puppetfile::Install do - let(:default_opts) { {root: "/some/nonexistent/path"} } - let(:puppetfile) { R10K::Puppetfile.new('/some/nonexistent/path', nil, nil) } + let(:default_opts) { { root: "/some/nonexistent/path" } } + let(:loader) { + R10K::ModuleLoader::Puppetfile.new( + basedir: '/some/nonexistent/path', + overrides: {force: false}) + } def installer(opts = {}, argv = [], settings = {}) opts = default_opts.merge(opts) @@ -11,8 +15,11 @@ end before(:each) do - allow(puppetfile).to receive(:load!).and_return(nil) - allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil, nil, nil).and_return(puppetfile) + allow(loader).to receive(:load!).and_return({}) + allow(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: {force: false, modules: {default_ref: nil}}}). + and_return(loader) end it_behaves_like "a puppetfile install action" @@ -20,14 +27,19 @@ describe "installing modules" do let(:modules) do (1..4).map do |idx| - R10K::Module::Base.new("author/modname#{idx}", "/some/nonexistent/path/modname#{idx}", nil) + R10K::Module::Base.new("author/modname#{idx}", + "/some/nonexistent/path/modname#{idx}", + {}) end end before do - allow(puppetfile).to receive(:purge!) - allow(puppetfile).to receive(:modules).and_return(modules) - allow(puppetfile).to receive(:modules_by_vcs_cachedir).and_return({none: modules}) + allow(loader).to receive(:load!).and_return({ + modules: modules, + managed_directories: [], + desired_contents: [], + purge_exclusions: [] + }) end it "syncs each module in the Puppetfile" do @@ -42,42 +54,67 @@ expect(installer.call).to eq false end - end - describe "purging" do - before do - allow(puppetfile).to receive(:modules).and_return([]) + it "reads in the default for git refs" do + modules.each { |m| expect(m).to receive(:sync) } + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: {force: false, modules: {default_ref: 'main'}}}). + and_return(loader) + + installer({}, [], {git: {default_ref: 'main'}}).call end + end + describe "purging" do it "purges the moduledir after installation" do - expect(puppetfile).to receive(:purge!) + allow(loader).to receive(:load!).and_return({ + modules: [], + desired_contents: [ 'root/foo' ], + managed_directories: [ 'root' ], + purge_exclusions: [ 'root/**/**.rb' ] + }) + + mock_cleaner = double("cleaner") + + expect(R10K::Util::Cleaner).to receive(:new). + with(["root"], ["root/foo"], ["root/**/**.rb"]). + and_return(mock_cleaner) + expect(mock_cleaner).to receive(:purge!) installer.call end end describe "using custom paths" do - it "can use a custom puppetfile path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/some/other/path/Puppetfile", nil, nil).and_return(puppetfile) + it "can use a custom moduledir path" do + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: {force: false, modules: {default_ref: nil}}, + puppetfile: "/some/other/path/Puppetfile"}). + and_return(loader) installer({puppetfile: "/some/other/path/Puppetfile"}).call - end - it "can use a custom moduledir path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", "/some/other/path/site-modules", nil, nil, nil).and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: {force: false, modules: {default_ref: nil}}, + moduledir: "/some/other/path/site-modules"}). + and_return(loader) installer({moduledir: "/some/other/path/site-modules"}).call end end describe "forcing to overwrite local changes" do - before do - allow(puppetfile).to receive(:modules).and_return([]) - end - it "can use the force overwrite option" do - subject = described_class.new({root: "/some/nonexistent/path", force: true}, []) - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil, nil, true).and_return(puppetfile) + allow(loader).to receive(:load!).and_return({ modules: [] }) + + subject = described_class.new({root: "/some/nonexistent/path", force: true}, [], {}) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + overrides: {force: true, modules: {default_ref: nil}}}). + and_return(loader) subject.call end diff -Nru r10k-3.7.0/spec/unit/action/puppetfile/purge_spec.rb r10k-4.0.0/spec/unit/action/puppetfile/purge_spec.rb --- r10k-3.7.0/spec/unit/action/puppetfile/purge_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/action/puppetfile/purge_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,7 +3,15 @@ describe R10K::Action::Puppetfile::Purge do let(:default_opts) { {root: "/some/nonexistent/path"} } - let(:puppetfile) { instance_double('R10K::Puppetfile', :load! => nil) } + let(:puppetfile) do + instance_double('R10K::ModuleLoader::Puppetfile', + :load! => { + :modules => %w{mod}, + :managed_directories => %w{foo}, + :desired_contents => %w{bar}, + :purge_exclusions => %w{baz} + }) + end def purger(opts = {}, argv = [], settings = {}) opts = default_opts.merge(opts) @@ -11,30 +19,40 @@ end before(:each) do - allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil).and_return(puppetfile) + allow(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path"}). + and_return(puppetfile) end it_behaves_like "a puppetfile action" it "purges unmanaged entries in the Puppetfile moduledir" do - expect(puppetfile).to receive(:purge!) + mock_cleaner = double("cleaner") + + expect(R10K::Util::Cleaner).to receive(:new). + with(["foo"], ["bar"], ["baz"]). + and_return(mock_cleaner) + + expect(mock_cleaner).to receive(:purge!) purger.call end describe "using custom paths" do - before(:each) do - allow(puppetfile).to receive(:purge!) - end - it "can use a custom puppetfile path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/some/other/path/Puppetfile").and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + puppetfile: "/some/other/path/Puppetfile"}). + and_return(puppetfile) purger({puppetfile: "/some/other/path/Puppetfile"}).call end it "can use a custom moduledir path" do - expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", "/some/other/path/site-modules", nil).and_return(puppetfile) + expect(R10K::ModuleLoader::Puppetfile).to receive(:new). + with({basedir: "/some/nonexistent/path", + moduledir: "/some/other/path/site-modules"}). + and_return(puppetfile) purger({moduledir: "/some/other/path/site-modules"}).call end diff -Nru r10k-3.7.0/spec/unit/action/runner_spec.rb r10k-4.0.0/spec/unit/action/runner_spec.rb --- r10k-3.7.0/spec/unit/action/runner_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/action/runner_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -10,11 +10,12 @@ Class.new do attr_reader :opts attr_reader :argv + attr_reader :settings def initialize(opts, argv, settings = {}) @opts = opts @argv = argv - @settings = {} + @settings = settings end def call @@ -158,55 +159,248 @@ end describe "configuring logging" do + before(:each) do + R10K::Logging.outputters.clear + end + it "sets the log level if :loglevel is provided" do runner = described_class.new({:opts => :yep, :loglevel => 'FATAL'}, %w[args yes], action_class) - expect(R10K::Logging).to receive(:level=).with('FATAL') + # The settings/overrides system causes the level to be set twice + expect(R10K::Logging).to receive(:level=).with('FATAL').twice runner.call end + # The logging fixture tests require a platform with syslog + if !R10K::Util::Platform.windows? + it "sets the log level if the logging.level setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + expect(R10K::Logging).to receive(:level=).with('FATAL') + runner.call + end + + it "sets the outputters if logging.outputs is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml' }, %w[args yes], action_class) + expect(R10K::Logging).to receive(:add_outputters).with([ + { type: 'file', parameters: { filename: 'r10k.log' } }, + { type: 'syslog' } + ]) + runner.call + end + + it "disables the default outputter if the logging.disable_default_stderr setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + expect(R10K::Logging).to receive(:disable_default_stderr=).with(true) + runner.call + end + + it "adds additional log outputs if the logging.outputs setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + runner.call + expect(R10K::Logging.outputters).to_not be_empty + end + + it "disables the default output if the logging.disable_default_stderr setting is provided" do + runner = described_class.new({ opts: :yep, config: 'spec/fixtures/unit/action/r10k_logging.yaml'}, %w[args yes], action_class) + runner.call + expect(runner.logger.outputters).to satisfy { |outputs| outputs.any? { |output| output.is_a?(R10K::Logging::TerminalOutputter) && output.level == Log4r::OFF } } + end + end + + it "doesn't add additional log outputs if the logging.outputs setting is not provided" do + runner.call + expect(R10K::Logging.outputters).to be_empty + end + + it "includes the default stderr outputter" do + runner.call + expect(runner.logger.outputters).to satisfy { |outputs| outputs.any? { |output| output.is_a? R10K::Logging::TerminalOutputter } } + end + it "does not modify the loglevel if :loglevel is not provided" do expect(R10K::Logging).to_not receive(:level=) runner.call end end - describe "configuration authorization" do - context "when license is not present" do - before(:each) do - expect(R10K::Util::License).to receive(:load).and_return(nil) - end + describe "configuring github app credentials" do + it 'errors if app id is passed without ssl key' do + runner = described_class.new( + { 'github-app-id': '/nonexistent', }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Must specify both id and SSL private key/) + end + + it 'errors if ssl key is passed without app id' do + runner = described_class.new( + { 'github-app-key': '/nonexistent', }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Must specify both id and SSL private key/) + end + + it 'errors if both app id and token paths are passed' do + runner = described_class.new( + { 'github-app-id': '/nonexistent', 'oauth-token': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'errors if both ssl key and token paths are passed' do + runner = described_class.new( + { 'github-app-key': '/nonexistent', 'oauth-token': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'errors if both ssl key and ssh key paths are passed' do + runner = described_class.new( + { 'github-app-key': '/nonexistent', 'private-key': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'errors if both app id and ssh key are passed' do + runner = described_class.new( + { 'github-app-id': '/nonexistent', 'private-key': '/also/fake' }, + %w[args yes], + action_class + ) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'saves the parameters in settings hash' do + runner = described_class.new( + { 'github-app-id': '123456', 'github-app-key': '/my/ssl/key', 'github-app-ttl': '600' }, + %w[args yes], + action_class + ) + runner.call + expect(runner.instance.settings[:git][:github_app_id]).to eq('123456') + expect(runner.instance.settings[:git][:github_app_key]).to eq('/my/ssl/key') + expect(runner.instance.settings[:git][:github_app_ttl]).to eq('600') + end - it "does not set authorization header on connection class" do - expect(PuppetForge::Connection).not_to receive(:authorization=) - runner.setup_authorization - end + it 'saves the parameters in settings hash without ttl and uses its default value' do + runner = described_class.new( + { 'github-app-id': '123456', 'github-app-key': '/my/ssl/key', }, + %w[args yes], + action_class + ) + runner.call + expect(runner.instance.settings[:git][:github_app_id]).to eq('123456') + expect(runner.instance.settings[:git][:github_app_key]).to eq('/my/ssl/key') + expect(runner.instance.settings[:git][:github_app_ttl]).to eq('120') end + end - context "when license is present but invalid" do - before(:each) do - expect(R10K::Util::License).to receive(:load).and_raise(R10K::Error.new('invalid license')) + describe "configuring git credentials" do + it 'errors if both token and key paths are passed' do + runner = described_class.new({ 'oauth-token': '/nonexistent', + 'private-key': '/also/fake' }, %w[args yes], action_class) + expect{ runner.call }.to raise_error(R10K::Error, /Cannot specify both/) + end + + it 'saves the sshkey path in settings hash' do + runner = described_class.new({ 'private-key': '/my/ssh/key' }, %w[args yes], action_class) + runner.call + expect(runner.instance.settings[:git][:private_key]).to eq('/my/ssh/key') + end + + it 'overrides per-repo sshkey in settings hash' do + runner = described_class.new({ config: "spec/fixtures/unit/action/r10k_creds.yaml", + 'private-key': '/my/ssh/key' }, + %w[args yes], + action_class) + runner.call + expect(runner.instance.settings[:git][:private_key]).to eq('/my/ssh/key') + expect(runner.instance.settings[:git][:repositories].count).to eq(2) + runner.instance.settings[:git][:repositories].each do |repo_settings| + expect(repo_settings[:private_key]).to eq('/my/ssh/key') end + end - it "issues warning to logger" do - expect(runner.logger).to receive(:warn).with(/invalid license/) - runner.setup_authorization + it 'saves the token path in settings hash' do + runner = described_class.new({ 'oauth-token': '/my/token/path' }, %w[args yes], action_class) + runner.call + expect(runner.instance.settings[:git][:oauth_token]).to eq('/my/token/path') + end + + it 'overrides per-repo oauth token in settings hash' do + runner = described_class.new({ config: "spec/fixtures/unit/action/r10k_creds.yaml", + 'oauth-token': '/my/token' }, + %w[args yes], + action_class) + runner.call + expect(runner.instance.settings[:git][:oauth_token]).to eq('/my/token') + expect(runner.instance.settings[:git][:repositories].count).to eq(2) + runner.instance.settings[:git][:repositories].each do |repo_settings| + expect(repo_settings[:oauth_token]).to eq('/my/token') end + end + end - it "does not set authorization header on connection class" do - expect(PuppetForge::Connection).not_to receive(:authorization=) + describe "configuration authorization" do + context "settings auth" do + it "sets the configured token as the forge authorization header" do + options = { config: "spec/fixtures/unit/action/r10k_forge_auth.yaml" } + runner = described_class.new(options, %w[args yes], action_class) + + expect(PuppetForge).to receive(:host=).with('http://private-forge.com') + expect(PuppetForge::Connection).to receive(:authorization=).with('faketoken') + expect(PuppetForge::Connection).to receive(:authorization).and_return('faketoken') + expect(R10K::Util::License).not_to receive(:load) + runner.setup_settings runner.setup_authorization end end - context "when license is present and valid" do - before(:each) do - mock_license = double('pe-license', :authorization_token => 'test token') - expect(R10K::Util::License).to receive(:load).and_return(mock_license) + context "license auth" do + context "when license is not present" do + before(:each) do + expect(R10K::Util::License).to receive(:load).and_return(nil) + end + + it "does not set authorization header on connection class" do + expect(PuppetForge::Connection).not_to receive(:authorization=) + runner.setup_authorization + end end - it "sets authorization header on connection class" do - expect(PuppetForge::Connection).to receive(:authorization=).with('test token') - runner.setup_authorization + context "when license is present but invalid" do + before(:each) do + expect(R10K::Util::License).to receive(:load).and_raise(R10K::Error.new('invalid license')) + end + + it "issues warning to logger" do + expect(runner.logger).to receive(:warn).with(/invalid license/) + runner.setup_authorization + end + + it "does not set authorization header on connection class" do + expect(PuppetForge::Connection).not_to receive(:authorization=) + runner.setup_authorization + end + end + + context "when license is present and valid" do + before(:each) do + mock_license = double('pe-license', :authorization_token => 'test token') + expect(R10K::Util::License).to receive(:load).and_return(mock_license) + end + + it "sets authorization header on connection class" do + expect(PuppetForge::Connection).to receive(:authorization=).with('test token') + runner.setup_authorization + end end end end diff -Nru r10k-3.7.0/spec/unit/environment/base_spec.rb r10k-4.0.0/spec/unit/environment/base_spec.rb --- r10k-3.7.0/spec/unit/environment/base_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/environment/base_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,10 +3,13 @@ describe R10K::Environment::Base do - subject(:environment) { described_class.new('envname', '/some/imaginary/path', 'env_name', {}) } + let(:basepath) { '/some/imaginary/path' } + let(:envname) { 'env_name' } + let(:path) { File.join(basepath, envname) } + subject(:environment) { described_class.new('envname', basepath, envname, {}) } it "can return the fully qualified path" do - expect(environment.path).to eq(Pathname.new('/some/imaginary/path/env_name')) + expect(environment.path).to eq(Pathname.new(path)) end it "raises an exception when #sync is called" do @@ -49,45 +52,55 @@ describe "#purge_exclusions" do let(:mock_env) { instance_double("R10K::Environment::Base") } let(:mock_puppetfile) { instance_double("R10K::Puppetfile", :environment= => true, :environment => mock_env) } + let(:loader) do + instance_double("R10K::ModuleLoader::Puppetfile", + :environment= => nil, + :load => { :modules => @modules, + :managed_directories => @managed_dirs, + :desired_contents => @desired_contents, + :purge_exclusions => @purge_ex }) + end before(:each) do - allow(mock_puppetfile).to receive(:managed_directories).and_return([]) - allow(mock_puppetfile).to receive(:desired_contents).and_return([]) - allow(R10K::Puppetfile).to receive(:new).and_return(mock_puppetfile) + @modules = [] + @managed_dirs = [] + @desired_contents = [] + @purge_exclusions = [] end it "excludes .r10k-deploy.json" do + allow(R10K::ModuleLoader::Puppetfile).to receive(:new).and_return(loader) + subject.deploy + expect(subject.purge_exclusions).to include(/r10k-deploy\.json/) end it "excludes puppetfile managed directories" do - managed_dirs = [ + @managed_dirs = [ '/some/imaginary/path/env_name/modules', '/some/imaginary/path/env_name/data', ] - expect(mock_puppetfile).to receive(:managed_directories).and_return(managed_dirs) + allow(R10K::ModuleLoader::Puppetfile).to receive(:new).and_return(loader) + subject.deploy exclusions = subject.purge_exclusions - managed_dirs.each do |dir| + @managed_dirs.each do |dir| expect(exclusions).to include(dir) end end describe "puppetfile desired contents" do - let(:desired_contents) do - basedir = subject.path.to_s - - [ 'modules/apache', 'data/local/site' ].collect do |c| - File.join(basedir, c) - end - end before(:each) do - allow(File).to receive(:directory?).with(/^\/some\/imaginary\/path/).and_return(true) + @desired_contents = [ 'modules/apache', 'data/local/site' ].collect do |c| + File.join(path, c) + end - expect(mock_puppetfile).to receive(:desired_contents).and_return(desired_contents) + allow(File).to receive(:directory?).and_return true + allow(R10K::ModuleLoader::Puppetfile).to receive(:new).and_return(loader) + subject.deploy end it "excludes desired directory contents with glob" do diff -Nru r10k-3.7.0/spec/unit/environment/git_spec.rb r10k-4.0.0/spec/unit/environment/git_spec.rb --- r10k-3.7.0/spec/unit/environment/git_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/environment/git_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -9,12 +9,28 @@ '/some/nonexistent/environmentdir', 'gitref', { - :remote => 'git://git-server.site/my-repo.git', + :remote => 'https://git-server.site/my-repo.git', :ref => 'd026ea677116424d2968edb9cee8cbc24d09322b', } ) end + describe "initializing" do + subject do + described_class.new('name', '/dir', 'ref', { + :remote => 'url', + :ref => 'value', + :puppetfile_name => 'Puppetfile', + :moduledir => 'modules', + :modules => { }, + }) + end + + it "accepts valid base class initialization arguments" do + expect(subject.name).to eq 'name' + end + end + describe "storing attributes" do it "can return the environment name" do expect(subject.name).to eq 'myenv' @@ -29,7 +45,7 @@ end it "can return the environment remote" do - expect(subject.remote).to eq 'git://git-server.site/my-repo.git' + expect(subject.remote).to eq 'https://git-server.site/my-repo.git' end it "can return the environment ref" do @@ -62,9 +78,10 @@ describe "enumerating modules" do it "loads the Puppetfile and returns modules in that puppetfile" do - expect(subject.puppetfile).to receive(:load) - expect(subject.puppetfile).to receive(:modules).and_return [:modules] - expect(subject.modules).to eq([:modules]) + loaded = { desired_contents: [], managed_directories: [], purge_exclusions: [] } + mod = double('A module', :name => 'dbl') + expect(subject.loader).to receive(:load).and_return(loaded.merge(modules: [mod])) + expect(subject.modules).to eq([mod]) end end diff -Nru r10k-3.7.0/spec/unit/environment/name_spec.rb r10k-4.0.0/spec/unit/environment/name_spec.rb --- r10k-3.7.0/spec/unit/environment/name_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/environment/name_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -2,25 +2,69 @@ require 'r10k/environment/name' describe R10K::Environment::Name do + describe "strip_component" do + it "does not modify the given name when no strip_component is given" do + bn = described_class.new('myenv', source: 'source', prefix: false) + expect(bn.dirname).to eq 'myenv' + expect(bn.name).to eq 'myenv' + expect(bn.original_name).to eq 'myenv' + end + + it "removes the first occurance of a regex match when a regex is given" do + bn = described_class.new('myenv', source: 'source', prefix: false, strip_component: '/env/') + expect(bn.dirname).to eq 'my' + expect(bn.name).to eq 'my' + expect(bn.original_name).to eq 'myenv' + end + + it "does not modify the given name when there is no regex match" do + bn = described_class.new('myenv', source: 'source', prefix: false, strip_component: '/bar/') + expect(bn.dirname).to eq 'myenv' + expect(bn.name).to eq 'myenv' + expect(bn.original_name).to eq 'myenv' + end + + it "removes the given name's prefix when it matches strip_component" do + bn = described_class.new('env/prod', source: 'source', prefix: false, strip_component: 'env/') + expect(bn.dirname).to eq 'prod' + expect(bn.name).to eq 'prod' + expect(bn.original_name).to eq 'env/prod' + end + + it "raises an error when given an integer" do + expect { + described_class.new('env/prod', source: 'source', prefix: false, strip_component: 4) + }.to raise_error(%r{Improper.*"4"}) + end + end + describe "prefixing" do it "uses the branch name as the dirname when prefixing is off" do bn = described_class.new('mybranch', :source => 'source', :prefix => false) expect(bn.dirname).to eq 'mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end it "prepends the source name when prefixing is on" do bn = described_class.new('mybranch', :source => 'source', :prefix => true) expect(bn.dirname).to eq 'source_mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end it "prepends the prefix name when prefixing is overridden" do bn = described_class.new('mybranch', {:prefix => "bar", :sourcename => 'foo'}) expect(bn.dirname).to eq 'bar_mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end it "uses the branch name as the dirname when prefixing is nil" do bn = described_class.new('mybranch', {:prefix => nil, :sourcename => 'foo'}) expect(bn.dirname).to eq 'mybranch' + expect(bn.name).to eq 'mybranch' + expect(bn.original_name).to eq 'mybranch' end end @@ -121,6 +165,8 @@ it "replaces invalid characters in #{branch} with underscores" do bn = described_class.new(branch.dup, {:correct => true}) expect(bn.dirname).to eq branch.gsub(/\W/, '_') + expect(bn.name).to eq branch + expect(bn.original_name).to eq branch end end diff -Nru r10k-3.7.0/spec/unit/environment/plain_spec.rb r10k-4.0.0/spec/unit/environment/plain_spec.rb --- r10k-3.7.0/spec/unit/environment/plain_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/environment/plain_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,8 @@ +require 'spec_helper' +require 'r10k/environment' + +describe R10K::Environment::Plain do + it "initializes successfully" do + expect(described_class.new('envname', '/basedir', 'dirname', {})).to be_a_kind_of(described_class) + end +end diff -Nru r10k-3.7.0/spec/unit/environment/svn_spec.rb r10k-4.0.0/spec/unit/environment/svn_spec.rb --- r10k-3.7.0/spec/unit/environment/svn_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/environment/svn_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -16,6 +16,18 @@ let(:working_dir) { subject.working_dir } + describe "initializing" do + subject do + described_class.new('name', '/dir', 'ref', { + :puppetfile_name => 'Puppetfile', + }) + end + + it "accepts valid base class initialization arguments" do + expect(subject.name).to eq 'name' + end + end + describe "storing attributes" do it "can return the environment name" do expect(subject.name).to eq 'myenv' @@ -66,9 +78,10 @@ describe "enumerating modules" do it "loads the Puppetfile and returns modules in that puppetfile" do - expect(subject.puppetfile).to receive(:load) - expect(subject.puppetfile).to receive(:modules).and_return [:modules] - expect(subject.modules).to eq([:modules]) + loaded = { managed_directories: [], desired_contents: [], purge_exclusions: [] } + mod = double('A module', :name => 'dbl') + expect(subject.loader).to receive(:load).and_return(loaded.merge(modules: [mod])) + expect(subject.modules).to eq([mod]) end end diff -Nru r10k-3.7.0/spec/unit/environment/tarball_spec.rb r10k-4.0.0/spec/unit/environment/tarball_spec.rb --- r10k-3.7.0/spec/unit/environment/tarball_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/environment/tarball_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,45 @@ +require 'spec_helper' +require 'r10k/environment' + +describe R10K::Environment::Tarball do + let(:tgz_path) do + File.expand_path('spec/fixtures/tarball/tarball.tar.gz', PROJECT_ROOT) + end + + let(:checksum) { '36afcfc2378b8235902d6e647fce7479da6898354d620388646c595a1155ed67' } + let(:base_params) { { source: tgz_path, version: checksum, modules: { } } } + + subject { described_class.new('envname', '/some/imaginary/path', 'dirname', base_params) } + + describe "initializing" do + it "accepts valid base class initialization arguments" do + expect(subject.name).to eq 'envname' + end + end + + describe "storing attributes" do + it "can return the environment name" do + expect(subject.name).to eq 'envname' + end + + it "can return the environment basedir" do + expect(subject.basedir).to eq '/some/imaginary/path' + end + + it "can return the environment dirname" do + expect(subject.dirname).to eq 'dirname' + end + + it "can return the environment path" do + expect(subject.path.to_s).to eq '/some/imaginary/path/dirname' + end + + it "can return the environment source" do + expect(subject.tarball.source).to eq tgz_path + end + + it "can return the environment version" do + expect(subject.tarball.checksum).to eq checksum + end + end +end diff -Nru r10k-3.7.0/spec/unit/environment/with_modules_spec.rb r10k-4.0.0/spec/unit/environment/with_modules_spec.rb --- r10k-3.7.0/spec/unit/environment/with_modules_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/environment/with_modules_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,122 @@ +require 'spec_helper' +require 'r10k/environment' + +describe R10K::Environment::WithModules do + subject do + described_class.new( + 'release42', + '/some/nonexistent/environmentdir', + 'prefix_release42', + { + :type => 'plain', + :modules => { + 'puppetlabs-stdlib' => { local: true }, + 'puppetlabs-concat' => { local: true }, + 'puppetlabs-exec' => { local: true }, + } + }.merge(subject_params) + ) + end + + # Default no additional params + let(:subject_params) { {} } + + describe "dealing with module conflicts" do + context "with no module conflicts" do + it "validates when there are no conflicts" do + mod = instance_double('R10K::Module::Base', name: 'nonconflict', origin: :puppetfile) + expect(subject.module_conflicts?(mod)).to eq false + end + end + + context "with module conflicts and default behavior" do + it "does not raise an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect(subject.logger).to receive(:warn).with(/Puppetfile.*both define.*ignored/i) + expect(subject.module_conflicts?(mod)).to eq true + end + end + + context "with module conflicts and 'error' behavior" do + let(:subject_params) {{ :module_conflicts => 'error' }} + it "raises an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect { subject.module_conflicts?(mod) }.to raise_error(R10K::Error, /Puppetfile.*both define.*/i) + end + end + + context "with module conflicts and 'override' behavior" do + let(:subject_params) {{ :module_conflicts => 'override' }} + it "does not raise an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect(subject.logger).to receive(:debug).with(/Puppetfile.*both define.*ignored/i) + expect(subject.module_conflicts?(mod)).to eq true + end + end + + context "with module conflicts and invalid configuration" do + let(:subject_params) {{ :module_conflicts => 'batman' }} + it "raises an error" do + mod = instance_double('R10K::Module::Base', name: 'stdlib', origin: :puppetfile) + expect { subject.module_conflicts?(mod) }.to raise_error(R10K::Error, /Unexpected value.*module_conflicts.*/i) + end + end + end + + describe "modules method" do + it "returns the configured modules, and Puppetfile modules" do + loaded = { managed_directories: [], desired_contents: [], purge_exclusions: [] } + puppetfile_mod = instance_double('R10K::Module::Base', name: 'zebra') + expect(subject.loader).to receive(:load).and_return(loaded.merge(modules: [puppetfile_mod])) + returned_modules = subject.modules + expect(returned_modules.map(&:name).sort).to eq(%w[concat exec stdlib zebra]) + end + end + + describe "module options" do + let(:subject_params) {{ + :modules => { + 'hieradata' => { + :type => 'git', + :source => 'git@git.example.com:site_data.git', + :install_path => '' + }, + 'site_data_2' => { + :type => 'git', + :source => 'git@git.example.com:site_data.git', + :install_path => 'subdir' + }, + + } + }} + + it "should support empty install_path" do + modules = subject.modules + expect(modules[0].title).to eq 'hieradata' + expect(modules[0].path).to eq Pathname.new('/some/nonexistent/environmentdir/prefix_release42/hieradata') + + end + + it "should support install_path" do + modules = subject.modules + expect(modules[1].title).to eq 'site_data_2' + expect(modules[1].path).to eq Pathname.new('/some/nonexistent/environmentdir/prefix_release42/subdir/site_data_2') + end + + context "with invalid configuration" do + let(:subject_params) {{ + :modules => { + 'site_data_2' => { + :type => 'git', + :source => 'git@git.example.com:site_data.git', + :install_path => '/absolute_path_outside_of_containing_environment' + } + } + }} + + it "raises an error" do + expect{ subject.modules }.to raise_error(R10K::Error, /Environment cannot.*outside of containing environment.*/i) + end + end + end +end diff -Nru r10k-3.7.0/spec/unit/git/alternates_spec.rb r10k-4.0.0/spec/unit/git/alternates_spec.rb --- r10k-3.7.0/spec/unit/git/alternates_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/git/alternates_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -13,13 +13,13 @@ it "reads the alternates file and splits on lines" do expect(subject.file).to receive(:file?).and_return true expect(subject.file).to receive(:readlines).and_return([ - "/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git\n", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git\n", + "/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git\n", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git\n", ]) expect(subject.read).to eq([ - "/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", + "/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", ]) end @@ -33,17 +33,17 @@ describe "determining if an entry is already present" do before do allow(subject).to receive(:to_a).and_return([ - "/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", + "/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", ]) end it "is true if the element is in the array of read entries" do - expect(subject).to include("/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git") + expect(subject).to include("/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git") end it "is false if the element is not in the array of read entries" do - expect(subject).to_not include("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git") + expect(subject).to_not include("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git") end end @@ -52,7 +52,7 @@ describe "and the git objects/info directory does not exist" do it "raises an error when the parent directory does not exist" do expect { - subject.write(["/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + subject.write(["/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) }.to raise_error(R10K::Git::GitError,"Cannot write /some/nonexistent/path/.git/objects/info/alternates; parent directory does not exist") end end @@ -66,51 +66,51 @@ end it "creates the alternates file with the new entry when not present" do - subject.write(["/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) - expect(io.string).to eq("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git\n") + subject.write(["/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(io.string).to eq("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git\n") end it "rewrites the file with all alternate entries" do - subject.write(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + subject.write(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) expect(io.string).to eq(<<-EOD) -/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git -/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git -/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git +/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git +/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git +/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git EOD end end describe "appending a new alternate object entry" do it "re-writes the file with the new entry concatenated to the file" do - expect(subject).to receive(:to_a).and_return(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(subject).to receive(:to_a).and_return(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) - expect(subject).to receive(:write).with(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/vagrant/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(subject).to receive(:write).with(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/vagrant/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) - subject.add("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git") + subject.add("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git") end end end describe "conditionally appending a new alternate object entry" do before do - expect(subject).to receive(:read).and_return(%w[/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git]) + expect(subject).to receive(:read).and_return(%w[/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git]) end it "adds the entry and returns true when the entry doesn't exist" do - expect(subject).to receive(:write).with(["/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git", - "/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git"]) - expect(subject.add?("/tmp/.r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git")).to eq true + expect(subject).to receive(:write).with(["/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git", + "/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git"]) + expect(subject.add?("/tmp/.r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git")).to eq true end it "doesn't modify the file and returns false when the entry exists" do expect(subject).to_not receive(:write) - expect(subject.add?("/var/cache/r10k/git/git---github.com-puppetlabs-puppetlabs-apache.git")).to eq false + expect(subject.add?("/var/cache/r10k/git/https---github.com-puppetlabs-puppetlabs-apache.git")).to eq false end end end diff -Nru r10k-3.7.0/spec/unit/git/cache_spec.rb r10k-4.0.0/spec/unit/git/cache_spec.rb --- r10k-3.7.0/spec/unit/git/cache_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/git/cache_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -21,7 +21,8 @@ end end - subject { subclass.new('git://some/git/remote') } + let(:remote) { 'https://some/git/remote' } + subject { subclass.new(remote) } describe "updating the cache" do it "only updates the cache once" do diff -Nru r10k-3.7.0/spec/unit/git/rugged/cache_spec.rb r10k-4.0.0/spec/unit/git/rugged/cache_spec.rb --- r10k-3.7.0/spec/unit/git/rugged/cache_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/git/rugged/cache_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -5,7 +5,7 @@ require 'r10k/git/rugged/cache' end - subject(:cache) { described_class.new('git://some/git/remote') } + subject(:cache) { described_class.new('https://some/git/remote') } it "wraps a Rugged::BareRepository instance" do expect(cache.repo).to be_a_kind_of R10K::Git::Rugged::BareRepository @@ -26,4 +26,23 @@ expect(described_class.settings[:cache_root]).to eq '/some/path' end end + + describe "remote url updates" do + before do + allow(subject.repo).to receive(:exist?).and_return true + allow(subject.repo).to receive(:fetch) + allow(subject.repo).to receive(:remotes).and_return({ 'origin' => 'https://some/git/remote' }) + end + + it "does not update the URLs if they match" do + expect(subject.repo).to_not receive(:update_remote) + subject.sync! + end + + it "updates the remote URL if they do not match" do + allow(subject.repo).to receive(:remotes).and_return({ 'origin' => 'foo'}) + expect(subject.repo).to receive(:update_remote) + subject.sync! + end + end end diff -Nru r10k-3.7.0/spec/unit/git/rugged/credentials_spec.rb r10k-4.0.0/spec/unit/git/rugged/credentials_spec.rb --- r10k-3.7.0/spec/unit/git/rugged/credentials_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/git/rugged/credentials_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -10,7 +10,7 @@ subject { described_class.new(repo) } - after(:all) { R10K::Git.settings.reset! } + after(:each) { R10K::Git.settings.reset! } describe "determining the username" do before { R10K::Git.settings[:username] = "moderns" } @@ -39,6 +39,7 @@ it "prefers a per-repository SSH private key" do allow(File).to receive(:readable?).with("/etc/puppetlabs/r10k/ssh/tessier-ashpool-id_rsa").and_return true + R10K::Git.settings[:private_key] = "/etc/puppetlabs/r10k/ssh/id_rsa" R10K::Git.settings[:repositories] = [{ remote: "ssh://git@tessier-ashpool.freeside/repo.git", private_key: "/etc/puppetlabs/r10k/ssh/tessier-ashpool-id_rsa"}] creds = subject.get_ssh_key_credentials("ssh://git@tessier-ashpool.freeside/repo.git", nil) @@ -78,6 +79,111 @@ end end + describe "generating github app tokens" do + it 'errors if app id has invalid characters' do + expect { subject.github_app_token("123A567890", "fake", "300") + }.to raise_error(R10K::Git::GitError, /App id contains invalid characters/) + end + it 'errors if app ttl has invalid characters' do + expect { subject.github_app_token("123456", "fake", "abc") + }.to raise_error(R10K::Git::GitError, /Github App token ttl contains/) + end + it 'errors if private file does not exist' do + R10K::Git.settings[:github_app_key] = "/missing/token/file" + expect(File).to receive(:readable?).with(R10K::Git.settings[:github_app_key]).and_return false + expect { + subject.github_app_token("123456", R10K::Git.settings[:github_app_key], "300") + }.to raise_error(R10K::Git::GitError, /App key is missing or unreadable/) + end + it 'errors if file is not a valid SSL key' do + token_file = Tempfile.new('token') + token_file.write('my_token') + token_file.close + R10K::Git.settings[:github_app_key] = token_file.path + expect(File).to receive(:readable?).with(token_file.path).and_return true + expect { + subject.github_app_token("123456", R10K::Git.settings[:github_app_key], "300") + }.to raise_error(R10K::Git::GitError, /App key is not a valid SSL key/) + token_file.unlink + end + end + + describe "generating token credentials" do + it 'errors if token file does not exist' do + R10K::Git.settings[:oauth_token] = "/missing/token/file" + expect(File).to receive(:readable?).with("/missing/token/file").and_return false + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + expect { + subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + }.to raise_error(R10K::Git::GitError, /cannot load OAuth token/) + end + + it 'errors if the token on stdin is not a valid OAuth token' do + allow($stdin).to receive(:read).and_return("token") + R10K::Git.settings[:oauth_token] = "-" + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + expect { + subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + }.to raise_error(R10K::Git::GitError, /invalid characters/) + end + + it 'errors if the token in the file is not a valid OAuth token' do + token_file = Tempfile.new('token') + token_file.write('my bad \ntoken') + token_file.close + R10K::Git.settings[:oauth_token] = token_file.path + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + expect { + subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + }.to raise_error(R10K::Git::GitError, /invalid characters/) + end + + it 'prefers per-repo token file' do + token_file = Tempfile.new('token') + token_file.write('my_token') + token_file.close + R10K::Git.settings[:oauth_token] = "/do/not/use" + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git", + oauth_token: token_file.path }] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds).to be_a_kind_of(Rugged::Credentials::UserPassword) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + expect(creds.instance_variable_get(:@username)).to eq("x-oauth-token") + end + + it 'uses the token from a file as a password' do + token_file = Tempfile.new('token') + token_file.write('my_token') + token_file.close + R10K::Git.settings[:oauth_token] = token_file.path + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds).to be_a_kind_of(Rugged::Credentials::UserPassword) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + expect(creds.instance_variable_get(:@username)).to eq("x-oauth-token") + end + + it 'uses the token from stdin as a password' do + allow($stdin).to receive(:read).and_return("my_token") + R10K::Git.settings[:oauth_token] = '-' + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds).to be_a_kind_of(Rugged::Credentials::UserPassword) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + expect(creds.instance_variable_get(:@username)).to eq("x-oauth-token") + end + + it 'only reads the token in once' do + expect($stdin).to receive(:read).and_return("my_token").once + R10K::Git.settings[:oauth_token] = '-' + R10K::Git.settings[:repositories] = [{remote: "https://tessier-ashpool.freeside/repo.git"}] + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + creds = subject.get_plaintext_credentials("https://tessier-ashpool.freeside/repo.git", nil) + expect(creds.instance_variable_get(:@password)).to eq("my_token") + end + end + describe "generating default credentials" do it "generates the rugged default credential type" do creds = subject.get_default_credentials("https://azurediamond:hunter2@tessier-ashpool.freeside/repo.git", "azurediamond") diff -Nru r10k-3.7.0/spec/unit/git/shellgit/cache_spec.rb r10k-4.0.0/spec/unit/git/shellgit/cache_spec.rb --- r10k-3.7.0/spec/unit/git/shellgit/cache_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/git/shellgit/cache_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,7 +3,7 @@ describe R10K::Git::ShellGit::Cache do - subject { described_class.new('git://some/git/remote') } + subject { described_class.new('https://some/git/remote') } it "wraps a ShellGit::BareRepository instance" do expect(subject.repo).to be_a_kind_of R10K::Git::ShellGit::BareRepository diff -Nru r10k-3.7.0/spec/unit/git/stateful_repository_spec.rb r10k-4.0.0/spec/unit/git/stateful_repository_spec.rb --- r10k-3.7.0/spec/unit/git/stateful_repository_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/git/stateful_repository_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -4,7 +4,7 @@ describe R10K::Git::StatefulRepository do - let(:remote) { 'git://some.site/some-repo.git' } + let(:remote) { 'https://some.site/some-repo.git' } let(:ref) { '0.9.x' } subject { described_class.new(remote, '/some/nonexistent/basedir', 'some-dirname') } @@ -19,6 +19,11 @@ expect(subject.sync_cache?(ref)).to eq true end + it "is true if the ref is HEAD" do + expect(cache).to receive(:exist?).and_return true + expect(subject.sync_cache?('HEAD')).to eq true + end + it "is true if the ref is unresolvable" do expect(cache).to receive(:exist?).and_return true expect(cache).to receive(:ref_type).with('0.9.x').and_return(:unknown) diff -Nru r10k-3.7.0/spec/unit/module/base_spec.rb r10k-4.0.0/spec/unit/module/base_spec.rb --- r10k-3.7.0/spec/unit/module/base_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/module/base_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -4,39 +4,92 @@ describe R10K::Module::Base do describe "parsing the title" do it "parses titles with no owner" do - m = described_class.new('eight_hundred', '/moduledir', []) + m = described_class.new('eight_hundred', '/moduledir', {}) expect(m.name).to eq 'eight_hundred' expect(m.owner).to be_nil end it "parses forward slash separated titles" do - m = described_class.new('branan/eight_hundred', '/moduledir', []) + m = described_class.new('branan/eight_hundred', '/moduledir', {}) expect(m.name).to eq 'eight_hundred' expect(m.owner).to eq 'branan' end it "parses hyphen separated titles" do - m = described_class.new('branan-eight_hundred', '/moduledir', []) + m = described_class.new('branan-eight_hundred', '/moduledir', {}) expect(m.name).to eq 'eight_hundred' expect(m.owner).to eq 'branan' end it "raises an error when the title is not correctly formatted" do expect { - described_class.new('branan!eight_hundred', '/moduledir', []) + described_class.new('branan!eight_hundred', '/moduledir', {}) }.to raise_error(ArgumentError, "Module name (branan!eight_hundred) must match either 'modulename' or 'owner/modulename'") end end + describe 'deleting the spec dir' do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + + before(:each) do + logger = double("logger") + allow_any_instance_of(described_class).to receive(:logger).and_return(logger) + allow(logger).to receive(:debug2).with(any_args) + allow(logger).to receive(:info).with(any_args) + end + + it 'removes the spec directory by default' do + FileUtils.mkdir_p(spec_path) + m = described_class.new(title, dirname, {}) + m.maybe_delete_spec_dir + expect(Dir.exist?(spec_path)).to eq false + end + + it 'detects a symlink and deletes the target' do + Dir.mkdir(dirname + module_name) + target_dir = Dir.mktmpdir + FileUtils.ln_s(target_dir, spec_path) + m = described_class.new(title, dirname, {}) + m.maybe_delete_spec_dir + expect(Dir.exist?(target_dir)).to eq false + end + + it 'does not remove the spec directory if overrides->modules->exclude_spec is set to false' do + FileUtils.mkdir_p(spec_path) + m = described_class.new(title, dirname, {overrides: {modules: {exclude_spec: false}}}) + m.maybe_delete_spec_dir + expect(Dir.exist?(spec_path)).to eq true + end + + it 'does not remove the spec directory if exclude_spec is set to false and overrides->modules->exclude_spec is true' do + FileUtils.mkdir_p(spec_path) + m = described_class.new(title, dirname, {exclude_spec: false, overrides: {modules: {exclude_spec: true}}}) + m.maybe_delete_spec_dir + expect(Dir.exist?(spec_path)).to eq true + end + + it 'does not remove the spec directory if spec_deletable is false' do + FileUtils.mkdir_p(spec_path) + m = described_class.new(title, dirname, {}) + m.spec_deletable = false + m.maybe_delete_spec_dir + expect(Dir.exist?(spec_path)).to eq true + end + end + describe "path variables" do it "uses the module name as the name" do - m = described_class.new('eight_hundred', '/moduledir', []) + m = described_class.new('eight_hundred', '/moduledir', {}) expect(m.dirname).to eq '/moduledir' expect(m.path).to eq(Pathname.new('/moduledir/eight_hundred')) end it "does not include the owner in the path" do - m = described_class.new('branan/eight_hundred', '/moduledir', []) + m = described_class.new('branan/eight_hundred', '/moduledir', {}) expect(m.dirname).to eq '/moduledir' expect(m.path).to eq(Pathname.new('/moduledir/eight_hundred')) end @@ -44,7 +97,7 @@ describe "with alternate variable names" do subject do - described_class.new('branan/eight_hundred', '/moduledir', []) + described_class.new('branan/eight_hundred', '/moduledir', {}) end it "aliases full_name to title" do @@ -61,7 +114,7 @@ end describe "accepting a visitor" do - subject { described_class.new('branan-eight_hundred', '/moduledir', []) } + subject { described_class.new('branan-eight_hundred', '/moduledir', {}) } it "passes itself to the visitor" do visitor = spy('visitor') diff -Nru r10k-3.7.0/spec/unit/module/forge_spec.rb r10k-4.0.0/spec/unit/module/forge_spec.rb --- r10k-3.7.0/spec/unit/module/forge_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/module/forge_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -9,22 +9,46 @@ let(:fixture_modulepath) { File.expand_path('spec/fixtures/module/forge', PROJECT_ROOT) } let(:empty_modulepath) { File.expand_path('spec/fixtures/empty', PROJECT_ROOT) } + describe "statically determined version support" do + it 'returns explicitly released forge versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: '8.0.0' }) + expect(static_version).to eq('8.0.0') + end + + it 'returns explicit pre-released forge versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: '8.0.0-pre1' }) + expect(static_version).to eq('8.0.0-pre1') + end + + it 'retuns nil for latest versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: :latest }) + expect(static_version).to eq(nil) + end + + it 'retuns nil for undefined versions' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { version: nil }) + expect(static_version).to eq(nil) + end + end + describe "implementing the Puppetfile spec" do it "should implement 'branan/eight_hundred', '8.0.0'" do - expect(described_class).to be_implement('branan/eight_hundred', '8.0.0') + expect(described_class).to be_implement('branan/eight_hundred', { type: 'forge', version: '8.0.0' }) end it "should implement 'branan-eight_hundred', '8.0.0'" do - expect(described_class).to be_implement('branan-eight_hundred', '8.0.0') + expect(described_class).to be_implement('branan-eight_hundred', { type: 'forge', version: '8.0.0' }) end + end - it "should fail with an invalid title" do - expect(described_class).to_not be_implement('branan!eight_hundred', '8.0.0') + describe "implementing the standard options interface" do + it "should implement {type: forge}" do + expect(described_class).to be_implement('branan-eight_hundred', { type: 'forge', version: '8.0.0', source: 'not implemented' }) end end describe "setting attributes" do - subject { described_class.new('branan/eight_hundred', '/moduledir', '8.0.0') } + subject { described_class.new('branan/eight_hundred', '/moduledir', { version: '8.0.0' }) } it "sets the name" do expect(subject.name).to eq 'eight_hundred' @@ -43,8 +67,14 @@ end end + describe "invalid attributes" do + it "errors on invalid versions" do + expect { described_class.new('branan/eight_hundred', '/moduledir', { version: '_8.0.0_' }) }.to raise_error ArgumentError, /version/ + end + end + describe "properties" do - subject { described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') } + subject { described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) } it "sets the module type to :forge" do expect(subject.properties).to include(:type => :forge) @@ -61,7 +91,7 @@ end context "when a module is deprecated" do - subject { described_class.new('puppetlabs/corosync', fixture_modulepath, :latest) } + subject { described_class.new('puppetlabs/corosync', fixture_modulepath, { version: :latest }) } it "warns on sync if module is not already insync" do allow(subject).to receive(:status).and_return(:absent) @@ -71,6 +101,8 @@ logger_dbl = double(Log4r::Logger) allow_any_instance_of(described_class).to receive(:logger).and_return(logger_dbl) + allow(logger_dbl).to receive(:info).with(/Deploying module to.*/) + allow(logger_dbl).to receive(:debug2).with(/No spec dir detected/) expect(logger_dbl).to receive(:warn).with(/puppet forge module.*puppetlabs-corosync.*has been deprecated/i) subject.sync @@ -82,6 +114,8 @@ logger_dbl = double(Log4r::Logger) allow_any_instance_of(described_class).to receive(:logger).and_return(logger_dbl) + allow(logger_dbl).to receive(:info).with(/Deploying module to.*/) + allow(logger_dbl).to receive(:debug2).with(/No spec dir detected/) expect(logger_dbl).to_not receive(:warn).with(/puppet forge module.*puppetlabs-corosync.*has been deprecated/i) subject.sync @@ -90,20 +124,27 @@ describe '#expected_version' do it "returns an explicitly given expected version" do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) expect(subject.expected_version).to eq '8.0.0' end it "uses the latest version from the forge when the version is :latest" do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, :latest) - expect(subject.v3_module).to receive_message_chain(:current_release, :version).and_return('8.8.8') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: :latest }) + release = double("Module Release", version: '8.8.8') + expect(subject.v3_module).to receive(:current_release).and_return(release).twice expect(subject.expected_version).to eq '8.8.8' end + + it "throws when there are no available versions" do + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: :latest }) + expect(subject.v3_module).to receive(:current_release).and_return(nil) + expect { subject.expected_version }.to raise_error(PuppetForge::ReleaseNotFound) + end end describe "determining the status" do - subject { described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') } + subject { described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) } it "is :absent if the module directory is absent" do allow(subject).to receive(:exist?).and_return false @@ -148,7 +189,24 @@ end describe "#sync" do - subject { described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') } + subject { described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) } + + context "syncing the repo" do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + subject { described_class.new(title, dirname, {}) } + + it 'defaults to deleting the spec dir' do + FileUtils.mkdir_p(spec_path) + expect(subject).to receive(:status).and_return(:absent) + expect(subject).to receive(:install) + subject.sync + expect(Dir.exist?(spec_path)).to eq false + end + end it 'does nothing when the module is in sync' do allow(subject).to receive(:status).and_return :insync @@ -156,31 +214,37 @@ expect(subject).to receive(:install).never expect(subject).to receive(:upgrade).never expect(subject).to receive(:reinstall).never - subject.sync + expect(subject.sync).to be false end it 'reinstalls the module when it is mismatched' do allow(subject).to receive(:status).and_return :mismatched expect(subject).to receive(:reinstall) - subject.sync + expect(subject.sync).to be true end it 'upgrades the module when it is outdated' do allow(subject).to receive(:status).and_return :outdated expect(subject).to receive(:upgrade) - subject.sync + expect(subject.sync).to be true end it 'installs the module when it is absent' do allow(subject).to receive(:status).and_return :absent expect(subject).to receive(:install) - subject.sync + expect(subject.sync).to be true + end + + it 'returns false if `should_sync?` is false' do + # modules do not sync if they are not requested + mod = described_class.new('my_org/my_mod', '/path/to/mod', { overrides: { modules: { requested_modules: ['other_mod'] } } }) + expect(mod.sync).to be false end end describe '#install' do it 'installs the module from the forge' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) release = instance_double('R10K::Forge::ModuleRelease') expect(R10K::Forge::ModuleRelease).to receive(:new).with('branan-eight_hundred', '8.0.0').and_return(release) expect(release).to receive(:install).with(subject.path) @@ -190,7 +254,7 @@ describe '#uninstall' do it 'removes the module path' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) expect(FileUtils).to receive(:rm_rf).with(subject.path.to_s) subject.uninstall end @@ -198,7 +262,7 @@ describe '#reinstall' do it 'uninstalls and then installs the module' do - subject = described_class.new('branan/eight_hundred', fixture_modulepath, '8.0.0') + subject = described_class.new('branan/eight_hundred', fixture_modulepath, { version: '8.0.0' }) expect(subject).to receive(:uninstall) expect(subject).to receive(:install) subject.reinstall diff -Nru r10k-3.7.0/spec/unit/module/git_spec.rb r10k-4.0.0/spec/unit/module/git_spec.rb --- r10k-3.7.0/spec/unit/module/git_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/module/git_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -10,15 +10,41 @@ allow(R10K::Git::StatefulRepository).to receive(:new).and_return(mock_repo) end + + describe "statically determined version support" do + it 'returns a given commit' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', commit: '123adf' }) + expect(static_version).to eq('123adf') + end + + it 'returns a given tag' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', tag: 'v1.2.3' }) + expect(static_version).to eq('v1.2.3') + end + + it 'returns a ref if it looks like a full commit sha' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', ref: '1234567890abcdef1234567890abcdef12345678' }) + expect(static_version).to eq('1234567890abcdef1234567890abcdef12345678') + end + + it 'returns nil for any non-sha-like ref' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', ref: 'refs/heads/main' }) + expect(static_version).to eq(nil) + end + + it 'returns nil for branches' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { git: 'my/remote', branch: 'main' }) + expect(static_version).to eq(nil) + end + end + describe "setting the owner and name" do describe "with a title of 'branan/eight_hundred'" do subject do described_class.new( 'branan/eight_hundred', '/moduledir', - { - :git => 'git://git-server.site/branan/puppet-eight_hundred', - } + { :git => 'https://git-server.site/branan/puppet-eight_hundred' } ) end @@ -40,9 +66,7 @@ described_class.new( 'eight_hundred', '/moduledir', - { - :git => 'git://git-server.site/branan/puppet-eight_hundred', - } + { :git => 'https://git-server.site/branan/puppet-eight_hundred' } ) end @@ -62,11 +86,12 @@ describe "properties" do subject do - described_class.new('boolean', '/moduledir', {:git => 'git://git.example.com/adrienthebo/puppet-boolean'}) + described_class.new('boolean', '/moduledir', {:git => 'https://git.example.com/adrienthebo/puppet-boolean', + overrides: {modules: {default_ref: "main"}}}) end before(:each) do - allow(mock_repo).to receive(:resolve).with('master').and_return('abc123') + allow(mock_repo).to receive(:resolve).with('main').and_return('abc123') allow(mock_repo).to receive(:head).and_return('abc123') end @@ -75,7 +100,7 @@ end it "sets the expected version" do - expect(subject.properties).to include(:expected => 'master') + expect(subject.properties).to include(:expected => 'main') end it "sets the actual version to the revision when the revision is available" do @@ -89,20 +114,54 @@ end end + describe 'syncing the repo' do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + subject { described_class.new(title, dirname, {overrides: {modules: {default_ref: "main"}}}) } + + before(:each) do + allow(mock_repo).to receive(:resolve).with('main').and_return('abc123') + end + + it 'defaults to deleting the spec dir' do + FileUtils.mkdir_p(spec_path) + allow(mock_repo).to receive(:sync) + subject.sync + expect(Dir.exist?(spec_path)).to eq false + end + + it 'returns true if repo was updated' do + expect(mock_repo).to receive(:sync).and_return(true) + expect(subject.sync).to be true + end + + it 'returns false if repo was not updated (in-sync)' do + expect(mock_repo).to receive(:sync).and_return(false) + expect(subject.sync).to be false + end + + it 'returns false if `should_sync?` is false' do + # modules do not sync if they are not requested + mod = described_class.new(title, dirname, { overrides: { modules: { requested_modules: ['other_mod'] } } }) + expect(mod.sync).to be false + end + end + describe "determining the status" do subject do described_class.new( 'boolean', '/moduledir', - { - :git => 'git://git.example.com/adrienthebo/puppet-boolean' - } + { :git => 'https://git.example.com/adrienthebo/puppet-boolean' } ) end it "delegates to the repo" do - expect(subject).to receive(:version).and_return 'master' - expect(mock_repo).to receive(:status).with('master').and_return :some_status + expect(subject).to receive(:version).and_return 'main' + expect(mock_repo).to receive(:status).with('main').and_return :some_status expect(subject.status).to eq(:some_status) end @@ -113,29 +172,35 @@ described_class.new('boolean', '/moduledir', base_opts.merge(extra_opts), env) end - let(:base_opts) { { git: 'git://git.example.com/adrienthebo/puppet-boolean' } } + let(:base_opts) { { git: 'https://git.example.com/adrienthebo/puppet-boolean' } } before(:each) do allow(mock_repo).to receive(:head).and_return('abc123') end - context "when option is unrecognized" do - let(:opts) { { unrecognized: true } } - - it "raises an error" do - expect { test_module(opts) }.to raise_error(ArgumentError, /unhandled options.*unrecognized/i) - end + it "raises an argument error when no refs are supplied" do + expect{test_module({}).properties}.to raise_error(ArgumentError, /unable.*desired ref.*no default/i) end - describe "desired ref" do - context "when no desired ref is given" do - it "defaults to master" do - expect(mock_repo).to receive(:resolve).with('master').and_return('abc123') + describe 'the overrides->modules->default_ref' do + context 'specifying a default_ref only' do + let(:opts) { {overrides: {modules: {default_ref: 'cranberry'}}} } + it "sets the expected ref to default_ref" do + expect(mock_repo).to receive(:resolve).with('cranberry').and_return('def456') + expect(test_module(opts).properties).to include(expected: 'cranberry') + end + end - expect(test_module({}).properties).to include(expected: 'master') + context 'specifying a default_ref and a default_branch' do + let(:opts) { {default_branch: 'orange', overrides: {modules: {default_ref: 'cranberry'}}}} + it "sets the expected ref to the default_branch" do + expect(mock_repo).to receive(:resolve).with('orange').and_return('def456') + expect(test_module(opts).properties).to include(expected: 'orange') end end + end + describe "desired ref" do context "specifying a static desired branch" do let(:opts) { { branch: 'banana' } } @@ -210,6 +275,14 @@ expect(mod.desired_ref).to eq(:control_branch) end + it "warns control branch may be unresolvable" do + logger = double("logger") + allow_any_instance_of(described_class).to receive(:logger).and_return(logger) + expect(logger).to receive(:warn).with(/Cannot track control repo branch.*boolean.*/) + + test_module(branch: :control_branch) + end + context "when default ref is provided and resolvable" do it "uses default ref" do expect(mock_repo).to receive(:resolve).with('default').and_return('abc123') @@ -268,6 +341,61 @@ end end end + + context "when using default_branch_override" do + before(:each) do + allow(mock_repo).to receive(:resolve).with(mock_env.ref).and_return(nil) + end + + context "and the default branch override is resolvable" do + it "uses the override" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return('5566aabb') + mod = test_module({branch: :control_branch, + default_branch: 'default', + default_branch_override: 'default_override'}, + mock_env) + expect(mod.properties).to include(expected: 'default_override') + end + end + + context "and the default branch override is not resolvable" do + context "and default branch is provided" do + it "falls back to the default" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return(nil) + expect(mock_repo).to receive(:resolve).with('default').and_return('5566aabb') + mod = test_module({branch: :control_branch, + default_branch: 'default', + default_branch_override: 'default_override'}, + mock_env) + expect(mod.properties).to include(expected: 'default') + end + end + + context "and default branch is not provided" do + it "raises the appropriate error" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return(nil) + mod = test_module({branch: :control_branch, + default_branch_override: 'default_override'}, + mock_env) + + expect { mod.properties }.to raise_error(ArgumentError, /unable to manage.*or resolve the default branch override.*no default provided/i) + end + end + + context "and default branch is not resolvable" do + it "raises the appropriate error" do + expect(mock_repo).to receive(:resolve).with('default_override').and_return(nil) + expect(mock_repo).to receive(:resolve).with('default').and_return(nil) + mod = test_module({branch: :control_branch, + default_branch: 'default', + default_branch_override: 'default_override'}, + mock_env) + + expect { mod.properties }.to raise_error(ArgumentError, /unable to manage.*or resolve the default branch override.*or resolve default/i) + end + end + end + end end end end diff -Nru r10k-3.7.0/spec/unit/module/svn_spec.rb r10k-4.0.0/spec/unit/module/svn_spec.rb --- r10k-3.7.0/spec/unit/module/svn_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/module/svn_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -6,6 +6,13 @@ include_context 'fail on execution' + describe "statically determined version support" do + it 'is unsupported by svn backed modules' do + static_version = described_class.statically_defined_version('branan/eight_hundred', { svn: 'my/remote', revision: '123adf' }) + expect(static_version).to eq(nil) + end + end + describe "determining it implements a Puppetfile mod" do it "implements mods with the :svn hash key" do implements = described_class.implement?('r10k-fixture-repo', :svn => 'https://github.com/adrienthebo/r10k-fixture-repo') @@ -119,6 +126,23 @@ end end + describe 'the default spec dir' do + let(:module_org) { "coolorg" } + let(:module_name) { "coolmod" } + let(:title) { "#{module_org}-#{module_name}" } + let(:dirname) { Pathname.new(Dir.mktmpdir) } + let(:spec_path) { dirname + module_name + 'spec' } + subject { described_class.new(title, dirname, {}) } + + it 'is deleted by default' do + FileUtils.mkdir_p(spec_path) + expect(subject).to receive(:status).and_return(:absent) + expect(subject).to receive(:install).and_return(nil) + subject.sync + expect(Dir.exist?(spec_path)).to eq false + end + end + describe "synchronizing" do subject { described_class.new('foo', '/moduledir', :svn => 'https://github.com/adrienthebo/r10k-fixture-repo', :rev => 123) } @@ -132,7 +156,7 @@ it "installs the SVN module" do expect(subject).to receive(:install) - subject.sync + expect(subject.sync).to be true end end @@ -142,14 +166,14 @@ it "reinstalls the module" do expect(subject).to receive(:reinstall) - subject.sync + expect(subject.sync).to be true end it "removes the existing directory" do expect(subject.path).to receive(:rmtree) allow(subject).to receive(:install) - subject.sync + expect(subject.sync).to be true end end @@ -159,7 +183,7 @@ it "upgrades the repository" do expect(subject).to receive(:update) - subject.sync + expect(subject.sync).to be true end end @@ -171,8 +195,14 @@ expect(subject).to receive(:reinstall).never expect(subject).to receive(:update).never - subject.sync + expect(subject.sync).to be false end end + + it 'and `should_sync?` is false' do + # modules do not sync if they are not requested + mod = described_class.new('my_mod', '/path/to/mod', { overrides: { modules: { requested_modules: ['other_mod'] } } }) + expect(mod.sync).to be false + end end end diff -Nru r10k-3.7.0/spec/unit/module/tarball_spec.rb r10k-4.0.0/spec/unit/module/tarball_spec.rb --- r10k-3.7.0/spec/unit/module/tarball_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/module/tarball_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,70 @@ +require 'spec_helper' +require 'r10k/module' +require 'fileutils' + +describe R10K::Module::Tarball do + include_context 'Tarball' + + let(:base_params) { { type: 'tarball', source: fixture_tarball, version: fixture_checksum } } + + subject do + described_class.new( + 'fixture-tarball', + moduledir, + base_params, + ) + end + + describe "setting the owner and name" do + describe "with a title of 'fixture-tarball'" do + it "sets the owner to 'fixture'" do + expect(subject.owner).to eq 'fixture' + end + + it "sets the name to 'tarball'" do + expect(subject.name).to eq 'tarball' + end + + it "sets the path to the given moduledir + modname" do + expect(subject.path.to_s).to eq(File.join(moduledir, 'tarball')) + end + end + end + + describe "properties" do + it "sets the module type to :tarball" do + expect(subject.properties).to include(type: :tarball) + end + + it "sets the version" do + expect(subject.properties).to include(expected: fixture_checksum) + end + end + + describe 'syncing the module' do + it 'defaults to deleting the spec dir' do + subject.sync + expect(Dir.exist?(File.join(moduledir, 'tarball', 'spec'))).to be(false) + end + end + + describe "determining the status" do + it "delegates to R10K::Tarball" do + expect(subject).to receive(:tarball).twice.and_return instance_double('R10K::Tarball', cache_valid?: true, insync?: true) + expect(subject).to receive(:path).twice.and_return instance_double('Pathname', exist?: true) + + expect(subject.status).to eq(:insync) + end + end + + describe "option parsing" do + describe "version" do + context "when no version is given" do + subject { described_class.new('fixture-tarball', moduledir, base_params.reject { |k| k.eql?(:version) }) } + it "does not require a version" do + expect(subject).to be_kind_of(described_class) + end + end + end + end +end diff -Nru r10k-3.7.0/spec/unit/module_loader/puppetfile_spec.rb r10k-4.0.0/spec/unit/module_loader/puppetfile_spec.rb --- r10k-3.7.0/spec/unit/module_loader/puppetfile_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/module_loader/puppetfile_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,439 @@ +require 'spec_helper' +require 'r10k/module_loader/puppetfile' +require 'tmpdir' + +describe R10K::ModuleLoader::Puppetfile do + describe 'initial parameters' do + describe 'honor' do + let(:options) do + { + basedir: '/test/basedir/env', + overrides: { modules: { deploy_modules: true } }, + environment: R10K::Environment::Git.new('env', + '/test/basedir/', + 'env', + { remote: 'https://foo/remote', + ref: 'env' }) + } + end + + subject { R10K::ModuleLoader::Puppetfile.new(**options) } + + describe 'the moduledir' do + it 'respects absolute paths' do + absolute_options = options.merge({moduledir: '/opt/puppetlabs/special/modules'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**absolute_options) + expect(puppetfile.instance_variable_get(:@moduledir)).to eq('/opt/puppetlabs/special/modules') + end + + it 'roots the moduledir in the basepath if a relative path is specified' do + relative_options = options.merge({moduledir: 'my/special/modules'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**relative_options) + expect(puppetfile.instance_variable_get(:@moduledir)).to eq('/test/basedir/env/my/special/modules') + end + end + + describe 'the Puppetfile' do + it 'respects absolute paths' do + absolute_options = options.merge({puppetfile: '/opt/puppetlabs/special/Puppetfile'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**absolute_options) + expect(puppetfile.instance_variable_get(:@puppetfile_path)).to eq('/opt/puppetlabs/special/Puppetfile') + end + + it 'roots the Puppetfile in the basepath if a relative path is specified' do + relative_options = options.merge({puppetfile: 'Puppetfile.global'}) + puppetfile = R10K::ModuleLoader::Puppetfile.new(**relative_options) + expect(puppetfile.instance_variable_get(:@puppetfile_path)).to eq('/test/basedir/env/Puppetfile.global') + end + end + + it 'the overrides' do + expect(subject.instance_variable_get(:@overrides)).to eq({ modules: { deploy_modules: true }}) + end + + it 'the environment' do + expect(subject.instance_variable_get(:@environment).name).to eq('env') + end + end + + describe 'sane defaults' do + subject { R10K::ModuleLoader::Puppetfile.new(basedir: '/test/basedir') } + + it 'has a moduledir rooted in the basedir' do + expect(subject.instance_variable_get(:@moduledir)).to eq('/test/basedir/modules') + end + + it 'has a Puppetfile rooted in the basedir' do + expect(subject.instance_variable_get(:@puppetfile_path)).to eq('/test/basedir/Puppetfile') + end + + it 'creates an empty overrides' do + expect(subject.instance_variable_get(:@overrides)).to eq({}) + end + + it 'does not require an environment' do + expect(subject.instance_variable_get(:@environment)).to eq(nil) + end + end + end + + describe 'adding modules' do + let(:basedir) { '/test/basedir' } + + subject { R10K::ModuleLoader::Puppetfile.new(basedir: basedir, + overrides: {modules: {exclude_spec: false}}) } + + it 'should transform Forge modules with a string arg to have a version key' do + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, hash_including(version: '1.2.3'), anything).and_call_original + + expect { subject.add_module('puppet/test_module', '1.2.3') }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should not accept Forge modules with a version comparison' do + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, hash_including(version: '< 1.2.0'), anything).and_call_original + + expect { + subject.add_module('puppet/test_module', '< 1.2.0') + }.to raise_error(ArgumentError, /module version .* is not a valid forge module version/i) + + expect(subject.modules.collect(&:name)).not_to include('test_module') + end + + it 'should not modify the overrides when adding modules' do + module_opts = { git: 'git@example.com:puppet/test_module.git' } + subject.add_module('puppet/test_module', module_opts) + expect(subject.instance_variable_get("@overrides")[:modules]).to eq({exclude_spec: false}) + end + + it 'should read the `exclude_spec` setting in the module definition and override the overrides' do + module_opts = { git: 'git@example.com:puppet/test_module.git', exclude_spec: true } + subject.add_module('puppet/test_module', module_opts) + expect(subject.modules[0].instance_variable_get("@exclude_spec")).to be true + end + + it 'should set :spec_deletable to true for modules in the basedir' do + module_opts = { git: 'git@example.com:puppet/test_module.git' } + subject.add_module('puppet/test_module', module_opts) + expect(subject.modules[0].spec_deletable).to be true + end + + it 'should set :spec_deletable to false for modules outside the basedir' do + module_opts = { git: 'git@example.com:puppet/test_module.git', install_path: 'some/path' } + subject.add_module('puppet/test_module', module_opts) + expect(subject.modules[0].spec_deletable).to be false + end + + it 'should accept non-Forge modules with a hash arg' do + module_opts = { git: 'git@example.com:puppet/test_module.git' } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, module_opts, anything).and_call_original + + expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should accept non-Forge modules with a valid relative :install_path option' do + module_opts = { + install_path: 'vendor', + git: 'git@example.com:puppet/test_module.git', + } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', File.join(basedir, 'vendor'), module_opts, anything).and_call_original + + expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should accept non-Forge modules with a valid absolute :install_path option' do + install_path = File.join(basedir, 'vendor') + + module_opts = { + install_path: install_path, + git: 'git@example.com:puppet/test_module.git', + } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', install_path, module_opts, anything).and_call_original + + expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } + expect(subject.modules.collect(&:name)).to include('test_module') + end + + it 'should reject non-Forge modules with an invalid relative :install_path option' do + module_opts = { + install_path: '../../vendor', + git: 'git@example.com:puppet/test_module.git', + } + + expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } + end + + it 'should reject non-Forge modules with an invalid absolute :install_path option' do + module_opts = { + install_path: '/tmp/mydata/vendor', + git: 'git@example.com:puppet/test_module.git', + } + + expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } + end + + it 'should disable and not add modules that conflict with the environment' do + env = instance_double('R10K::Environment::Base') + mod = instance_double('R10K::Module::Base', name: 'conflict', origin: :puppetfile, 'origin=': nil) + allow(env).to receive(:name).and_return('conflict') + loader = R10K::ModuleLoader::Puppetfile.new(basedir: basedir, environment: env) + allow(env).to receive(:'module_conflicts?').with(mod).and_return(true) + allow(mod).to receive(:spec_deletable=) + + expect(R10K::Module).to receive(:from_metadata).with('conflict', anything, anything, anything).and_return(mod) + expect { loader.add_module('conflict', {}) }.not_to change { loader.modules } + end + end + + describe '#purge_exclusions' do + let(:managed_dirs) { ['dir1', 'dir2'] } + subject { R10K::ModuleLoader::Puppetfile.new(basedir: '/test/basedir') } + + it 'includes managed_directories' do + expect(subject.send(:determine_purge_exclusions, managed_dirs)).to match_array(managed_dirs) + end + + context 'when belonging to an environment' do + let(:env_contents) { ['env1', 'env2' ] } + let(:env) { double(:environment, desired_contents: env_contents) } + before { + allow(env).to receive(:name).and_return('env1') + } + subject { R10K::ModuleLoader::Puppetfile.new(basedir: '/test/basedir', environment: env) } + + it "includes environment's desired_contents" do + expect(subject.send(:determine_purge_exclusions, managed_dirs)).to match_array(managed_dirs + env_contents) + end + end + end + + describe '#managed_directories' do + + let(:basedir) { '/test/basedir' } + subject { R10K::ModuleLoader::Puppetfile.new(basedir: basedir) } + + before do + allow(subject).to receive(:puppetfile_content).and_return('') + end + + it 'returns an array of paths that #purge! will operate within' do + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', subject.moduledir, hash_including(version: '1.2.3'), anything).and_call_original + subject.add_module('puppet/test_module', '1.2.3') + subject.load! + + expect(subject.modules.length).to be 1 + expect(subject.managed_directories).to match_array([subject.moduledir]) + end + + context "with a module with install_path == ''" do + it "basedir isn't in the list of paths to purge" do + module_opts = { install_path: '', git: 'git@example.com:puppet/test_module.git' } + + expect(R10K::Module).to receive(:from_metadata).with('puppet/test_module', basedir, module_opts, anything).and_call_original + subject.add_module('puppet/test_module', module_opts) + subject.load! + + expect(subject.modules.length).to be 1 + expect(subject.managed_directories).to be_empty + end + end + end + + describe 'evaluating a Puppetfile' do + def expect_wrapped_error(error, pf_path, error_type) + expect(error).to be_a_kind_of(R10K::Error) + expect(error.message).to eq("Failed to evaluate #{pf_path}") + expect(error.original).to be_a_kind_of(error_type) + end + + subject { described_class.new(basedir: @path) } + + it 'wraps and re-raises syntax errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'invalid-syntax') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, SyntaxError) + end + end + + it 'wraps and re-raises load errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'load-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, LoadError) + end + end + + it 'wraps and re-raises argument errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'argument-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, ArgumentError) + end + end + + describe 'forge declaration' do + before(:each) do + PuppetForge.host = "" + end + + it 'is respected if `allow_puppetfile_override` is true' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'forge-override') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, overrides: { forge: { allow_puppetfile_override: true } }) + puppetfile.load! + expect(PuppetForge.host).to eq("my.custom.forge.com/") + end + + it 'is ignored if `allow_puppetfile_override` is false' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'forge-override') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, overrides: { forge: { allow_puppetfile_override: false } }) + expect(PuppetForge).not_to receive(:host=).with("my.custom.forge.com") + puppetfile.load! + expect(PuppetForge.host).to eq("/") + end + end + + it 'rejects Puppetfiles with duplicate module names' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'duplicate-module-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error(R10K::Error, /Puppetfiles cannot contain duplicate module names/i) + end + + it 'wraps and re-raises name errors' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'name-error') + pf_path = File.join(@path, 'Puppetfile') + expect { + subject.load! + }.to raise_error do |e| + expect_wrapped_error(e, pf_path, NameError) + end + end + + it 'accepts a forge module with a version' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + pf_path = File.join(@path, 'Puppetfile') + expect { subject.load! }.not_to raise_error + end + + describe 'setting a custom moduledir' do + it 'allows setting an absolute moduledir' do + @path = '/fake/basedir' + allow(subject).to receive(:puppetfile_content).and_return('moduledir "/fake/moduledir"') + subject.load! + expect(subject.instance_variable_get(:@moduledir)).to eq('/fake/moduledir') + end + + it 'roots relative moduledirs in the basedir' do + @path = '/fake/basedir' + allow(subject).to receive(:puppetfile_content).and_return('moduledir "my/moduledir"') + subject.load! + expect(subject.instance_variable_get(:@moduledir)).to eq(File.join(@path, 'my/moduledir')) + end + end + + it 'accepts a forge module without a version' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-without-version') + pf_path = File.join(@path, 'Puppetfile') + expect { subject.load! }.not_to raise_error + end + + it 'creates a git module and applies the default branch specified in the Puppetfile' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') + pf_path = File.join(@path, 'Puppetfile') + expect { subject.load! }.not_to raise_error + git_module = subject.modules[0] + expect(git_module.default_ref).to eq 'here_lies_the_default_branch' + end + + it 'creates a git module and applies the provided default_branch_override' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') + pf_path = File.join(@path, 'Puppetfile') + default_branch_override = 'default_branch_override_name' + subject.default_branch_override = default_branch_override + expect { subject.load! }.not_to raise_error + git_module = subject.modules[0] + expect(git_module.default_override_ref).to eq default_branch_override + expect(git_module.default_ref).to eq 'here_lies_the_default_branch' + end + + describe 'using module metadata' do + it 'properly loads module metadata' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + metadata = subject.load_metadata[:modules].map { |mod| [ mod.name, mod.version ] }.to_h + expect(metadata['apt']).to eq('2.1.1') + expect(metadata['stdlib']).to eq(nil) + expect(metadata['concat']).to eq(nil) + expect(metadata['rpm']).to eq('2.1.1-pre1') + expect(metadata['foo']).to eq(nil) + expect(metadata['bar']).to eq('v1.2.3') + expect(metadata['baz']).to eq('123abc456') + expect(metadata['fizz']).to eq('1234567890abcdef1234567890abcdef12345678') + expect(metadata['buzz']).to eq(nil) + expect(metadata['canary']).to eq('0.0.0') + end + + it 'does not load module implementations for static versions unless the module install path does not exist on disk' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + subject.load_metadata + modules = subject.load[:modules].map { |mod| [ mod.name, mod ] }.to_h + expect(modules['apt']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['stdlib']).to be_a_kind_of(R10K::Module::Forge) + expect(modules['concat']).to be_a_kind_of(R10K::Module::Forge) + expect(modules['rpm']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['foo']).to be_a_kind_of(R10K::Module::Git) + expect(modules['bar']).to be_a_kind_of(R10K::Module::Git) + expect(modules['baz']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['fizz']).to be_a_kind_of(R10K::Module::Definition) + expect(modules['buzz']).to be_a_kind_of(R10K::Module::Git) + expect(modules['canary']).to be_a_kind_of(R10K::Module::Definition) + end + + it 'loads module implementations whose static versions are different' do + fixture_path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + @path = Dir.mktmpdir + unsynced_pf_path = File.join(fixture_path, 'Puppetfile') + FileUtils.cp(unsynced_pf_path, @path) + + subject.load_metadata + + synced_pf_path = File.join(fixture_path, 'Puppetfile.new') + FileUtils.cp(synced_pf_path, File.join(@path, 'Puppetfile')) + + modules = subject.load[:modules].map { |mod| [ mod.name, mod ] }.to_h + + expect(modules['apt']).to be_a_kind_of(R10K::Module::Forge) + end + + end + + describe 'using module-exclude-regex' do + it 'can exclude a module from being installed' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, module_exclude_regex: '^concat$') + puppetfile.load! + expect(puppetfile.modules.collect(&:name)).not_to include('concat') + end + + it 'can exclude multiple modules from being installed' do + @path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'various-modules') + puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: @path, module_exclude_regex: '^ba[rz]$') + puppetfile.load! + expect(puppetfile.modules.collect(&:name)).not_to include('bar') + expect(puppetfile.modules.collect(&:name)).not_to include('baz') + end + end + end +end diff -Nru r10k-3.7.0/spec/unit/module_spec.rb r10k-4.0.0/spec/unit/module_spec.rb --- r10k-3.7.0/spec/unit/module_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/module_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -3,27 +3,108 @@ describe R10K::Module do describe 'delegating to R10K::Module::Git' do - it "accepts args {:git => 'git url}" do - obj = R10K::Module.new('foo', '/modulepath', :git => 'git url') - expect(obj).to be_a_kind_of(R10K::Module::Git) + [ {git: 'git url'}, + {type: 'git', source: 'git url'}, + ].each do |scenario| + it "accepts a name matching 'test' and args #{scenario.inspect}" do + obj = R10K::Module.new('test', '/modulepath', scenario) + expect(obj).to be_a_kind_of(R10K::Module::Git) + expect(obj.send(:instance_variable_get, :'@remote')).to eq('git url') + end + end + end + + describe 'delegating to R10K::Module::Svn' do + [ {svn: 'svn url'}, + {type: 'svn', source: 'svn url'}, + ].each do |scenario| + it "accepts a name matching 'test' and args #{scenario.inspect}" do + obj = R10K::Module.new('test', '/modulepath', scenario) + expect(obj).to be_a_kind_of(R10K::Module::SVN) + expect(obj.send(:instance_variable_get, :'@url')).to eq('svn url') + end end end describe 'delegating to R10K::Module::Forge' do - [ - ['bar/quux', nil], - ['bar-quux', nil], - ['bar/quux', '8.0.0'], + [ 'bar/quux', + 'bar-quux', + ].each do |scenario| + it "accepts a name matching #{scenario} and version nil" do + obj = R10K::Module.new(scenario, '/modulepath', { type: 'forge', version: nil }) + expect(obj).to be_a_kind_of(R10K::Module::Forge) + end + end + [ {type: 'forge', version: '8.0.0'}, ].each do |scenario| - it "accepts a name matching #{scenario[0]} and args #{scenario[1].inspect}" do - expect(R10K::Module.new(scenario[0], '/modulepath', scenario[1])).to be_a_kind_of(R10K::Module::Forge) + it "accepts a name matching bar-quux and args #{scenario.inspect}" do + obj = R10K::Module.new('bar-quux', '/modulepath', scenario) + expect(obj).to be_a_kind_of(R10K::Module::Forge) + expect(obj.send(:instance_variable_get, :'@expected_version')).to eq('8.0.0') + end + end + + describe 'when the module is ostensibly on disk' do + before do + owner = 'theowner' + module_name = 'themodulename' + @title = "#{owner}-#{module_name}" + metadata = <<~METADATA + { + "name": "#{@title}", + "version": "1.2.0" + } + METADATA + @dirname = Dir.mktmpdir + module_path = File.join(@dirname, module_name) + FileUtils.mkdir(module_path) + File.open("#{module_path}/metadata.json", 'w') do |file| + file.write(metadata) + end + end + + it 'sets the expected version to what is found in the metadata' do + obj = R10K::Module.new(@title, @dirname, {type: 'forge', version: nil}) + expect(obj.send(:instance_variable_get, :'@expected_version')).to eq('1.2.0') end end end it "raises an error if delegation fails" do expect { - R10K::Module.new('bar!quux', '/modulepath', ["NOPE NOPE NOPE NOPE!"]) + R10K::Module.new('bar!quux', '/modulepath', {version: ["NOPE NOPE NOPE NOPE!"]}) }.to raise_error RuntimeError, /doesn't have an implementation/ end + + describe 'Given a set of initialization parameters for R10K::Module' do + [ ['name', {git: 'git url'}], + ['name', {type: 'git', source: 'git url'}], + ['name', {svn: 'svn url'}], + ['name', {type: 'svn', source: 'svn url'}], + ['namespace-name', {type: 'forge', version: '8.0.0'}] + ].each do |(name, options)| + it 'can handle the default_branch_override option' do + expect { + obj = R10K::Module.new(name, '/modulepath', options.merge({default_branch_override: 'foo'})) + expect(obj).to be_a_kind_of(R10K::Module::Base) + }.not_to raise_error + end + describe 'the exclude_spec setting' do + it 'sets the exclude_spec instance variable to true by default' do + obj = R10K::Module.new(name, '/modulepath', options) + expect(obj.instance_variable_get("@exclude_spec")).to eq(true) + end + it 'cannot be overridden by the settings from the cli, r10k.yaml, or settings default' do + options = options.merge({exclude_spec: false, overrides: {modules: {exclude_spec: true}}}) + obj = R10K::Module.new(name, '/modulepath', options) + expect(obj.instance_variable_get("@exclude_spec")).to eq(false) + end + it 'reads the setting from the cli, r10k.yaml, or settings default when not provided directly' do + options = options.merge({overrides: {modules: {exclude_spec: false}}}) + obj = R10K::Module.new(name, '/modulepath', options) + expect(obj.instance_variable_get("@exclude_spec")).to eq(false) + end + end + end + end end diff -Nru r10k-3.7.0/spec/unit/puppetfile_spec.rb r10k-4.0.0/spec/unit/puppetfile_spec.rb --- r10k-3.7.0/spec/unit/puppetfile_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/puppetfile_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -6,13 +6,11 @@ subject do described_class.new( '/some/nonexistent/basedir', - nil, - nil, - 'Puppetfile.r10k' + {puppetfile_name: 'Puppetfile.r10k'} ) end - describe "a custom puppetfile Puppetfile.r10k" do + describe "a custom puppetfile_name" do it "is the basedir joined with '/Puppetfile.r10k' path" do expect(subject.puppetfile_path).to eq '/some/nonexistent/basedir/Puppetfile.r10k' end @@ -22,10 +20,45 @@ describe R10K::Puppetfile do + describe "a custom relative puppetfile_path" do + it "is the basedir joined with the puppetfile_path" do + relative_subject = described_class.new('/some/nonexistent/basedir', + {puppetfile_path: 'relative/Puppetfile'}) + expect(relative_subject.puppetfile_path).to eq '/some/nonexistent/basedir/relative/Puppetfile' + end + end + + describe "a custom absolute puppetfile_path" do + it "is the puppetfile_path as given" do + absolute_subject = described_class.new('/some/nonexistent/basedir', + {puppetfile_path: '/some/absolute/custom/Puppetfile'}) + expect(absolute_subject.puppetfile_path).to eq '/some/absolute/custom/Puppetfile' + end + end +end + +describe R10K::Puppetfile do + subject do - described_class.new( - '/some/nonexistent/basedir' - ) + described_class.new( '/some/nonexistent/basedir', {}) + end + + describe "backwards compatibility with older calling conventions" do + it "honors all arguments correctly" do + puppetfile = described_class.new('/some/nonexistant/basedir', '/some/nonexistant/basedir/site-modules', nil, 'Pupupupetfile', true) + expect(puppetfile.force).to eq(true) + expect(puppetfile.moduledir).to eq('/some/nonexistant/basedir/site-modules') + expect(puppetfile.puppetfile_path).to eq('/some/nonexistant/basedir/Pupupupetfile') + expect(puppetfile.overrides).to eq({}) + end + + it "handles defaults correctly" do + puppetfile = described_class.new('/some/nonexistant/basedir', nil, nil, nil) + expect(puppetfile.force).to eq(false) + expect(puppetfile.moduledir).to eq('/some/nonexistant/basedir/modules') + expect(puppetfile.puppetfile_path).to eq('/some/nonexistant/basedir/Puppetfile') + expect(puppetfile.overrides).to eq({}) + end end describe "the default moduledir" do @@ -53,237 +86,147 @@ end end - describe "adding modules" do - it "should accept Forge modules with a string arg" do - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '1.2.3', anything).and_call_original - - expect { subject.add_module('puppet/test_module', '1.2.3') }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should not accept Forge modules with a version comparison" do - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '< 1.2.0', anything).and_call_original - - expect { - subject.add_module('puppet/test_module', '< 1.2.0') - }.to raise_error(RuntimeError, /module puppet\/test_module.*doesn't have an implementation/i) - - expect(subject.modules.collect(&:name)).not_to include('test_module') - end - - it "should accept non-Forge modules with a hash arg" do - module_opts = { git: 'git@example.com:puppet/test_module.git' } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, module_opts, anything).and_call_original - - expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should accept non-Forge modules with a valid relative :install_path option" do - module_opts = { - install_path: 'vendor', - git: 'git@example.com:puppet/test_module.git', - } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', File.join(subject.basedir, 'vendor'), module_opts, anything).and_call_original - - expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should accept non-Forge modules with a valid absolute :install_path option" do - install_path = File.join(subject.basedir, 'vendor') - - module_opts = { - install_path: install_path, - git: 'git@example.com:puppet/test_module.git', - } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', install_path, module_opts, anything).and_call_original + describe "loading a Puppetfile" do + context 'using load' do + it "returns the loaded content" do + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - expect { subject.add_module('puppet/test_module', module_opts) }.to change { subject.modules } - expect(subject.modules.collect(&:name)).to include('test_module') - end - - it "should reject non-Forge modules with an invalid relative :install_path option" do - module_opts = { - install_path: '../../vendor', - git: 'git@example.com:puppet/test_module.git', - } - - allow(R10K::Module).to receive(:new).with('puppet/test_module', File.join(subject.basedir, 'vendor'), module_opts, anything).and_call_original - - expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } - end + loaded_content = subject.load + expect(loaded_content).to be_an_instance_of(Hash) - it "should reject non-Forge modules with an invalid absolute :install_path option" do - module_opts = { - install_path: '/tmp/mydata/vendor', - git: 'git@example.com:puppet/test_module.git', - } + has_some_data = loaded_content.values.none?(&:empty?) + expect(has_some_data).to be true + end - allow(R10K::Module).to receive(:new).with('puppet/test_module', File.join(subject.basedir, 'vendor'), module_opts, anything).and_call_original + it "handles a relative basedir" do + path = File.join('spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules } - end + loaded_content = subject.load + expect(loaded_content).to be_an_instance_of(Hash) - it "groups modules by vcs cache location" do - module_opts = { install_path: File.join(subject.basedir, 'vendor') } - opts1 = module_opts.merge(git: 'git@example.com:puppet/test_module.git') - opts2 = module_opts.merge(git: 'git@example.com:puppet/test_module_c.git') - sanitized_name1 = "git@example.com-puppet-test_module.git" - sanitized_name2 = "git@example.com-puppet-test_module_c.git" + has_some_data = loaded_content.values.none?(&:empty?) + expect(has_some_data).to be true + end - subject.add_module('puppet/test_module_a', opts1) - subject.add_module('puppet/test_module_b', opts1) - subject.add_module('puppet/test_module_c', opts2) - subject.add_module('puppet/test_module_d', '1.2.3') + it "is idempotent" do + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - mods_by_cachedir = subject.modules_by_vcs_cachedir + expect(subject.loader).to receive(:load!).and_call_original.once - expect(mods_by_cachedir[:none].length).to be 1 - expect(mods_by_cachedir[sanitized_name1].length).to be 2 - expect(mods_by_cachedir[sanitized_name2].length).to be 1 - end - end + loaded_content1 = subject.load + expect(subject.loaded?).to be true + loaded_content2 = subject.load - describe "#purge_exclusions" do - let(:managed_dirs) { ['dir1', 'dir2'] } + expect(loaded_content2).to eq(loaded_content1) + end - before(:each) do - allow(subject).to receive(:managed_directories).and_return(managed_dirs) + it "returns nil if Puppetfile doesn't exist" do + path = '/rando/path/that/wont/exist' + subject = described_class.new(path, {}) + expect(subject.load).to eq nil + end end - it "includes managed_directories" do - expect(subject.purge_exclusions).to match_array(managed_dirs) - end + context 'using load!' do + it "returns the loaded content" do + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {}) - context "when belonging to an environment" do - let(:env_contents) { ['env1', 'env2' ] } + loaded_content = subject.load! + expect(loaded_content).to be_an_instance_of(Hash) - before(:each) do - mock_env = double(:environment, desired_contents: env_contents) - allow(subject).to receive(:environment).and_return(mock_env) + has_some_data = loaded_content.values.none?(&:empty?) + expect(has_some_data).to be true end - it "includes environment's desired_contents" do - expect(subject.purge_exclusions).to match_array(managed_dirs + env_contents) + it "raises if Puppetfile doesn't exist" do + path = '/rando/path/that/wont/exist' + subject = described_class.new(path, {}) + expect { + subject.load! + }.to raise_error(/No such file or directory.*\/rando\/path\/.*/) end end end - describe '#managed_directories' do - it 'returns an array of paths that can be purged' do - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '1.2.3', anything).and_call_original - - subject.add_module('puppet/test_module', '1.2.3') - expect(subject.managed_directories).to match_array(["/some/nonexistent/basedir/modules"]) - end - - context 'with a module with install_path == \'\'' do - it 'basedir isn\'t in the list of paths to purge' do - module_opts = { install_path: '', git: 'git@example.com:puppet/test_module.git' } + describe 'default_branch_override' do + it 'is passed correctly to module loader init' do + # This path doesn't matter so long as it has a Puppetfile within it + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {overrides: {environments: {default_branch_override: 'foo'}}}) - allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.basedir, module_opts, anything).and_call_original + repo = instance_double('R10K::Git::StatefulRepository') + allow(repo).to receive(:resolve).with('foo').and_return(true) + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) - subject.add_module('puppet/test_module', module_opts) - expect(subject.managed_directories).to be_empty - end - end - end + allow(subject.loader).to receive(:puppetfile_content).and_return <<-EOPF + # Track control branch and fall-back to main if no matching branch. + mod 'hieradata', + :git => 'git@git.example.com:organization/hieradata.git', + :branch => :control_branch, + :default_branch => 'main' + EOPF - describe "evaluating a Puppetfile" do - def expect_wrapped_error(orig, pf_path, wrapped_error) - expect(orig).to be_a_kind_of(R10K::Error) - expect(orig.message).to eq("Failed to evaluate #{pf_path}") - expect(orig.original).to be_a_kind_of(wrapped_error) - end + expect(subject.logger).not_to receive(:warn). + with(/Mismatch between passed and initialized.*preferring passed value/) - it "wraps and re-raises syntax errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'invalid-syntax') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, SyntaxError) - end - end + subject.load - it "wraps and re-raises load errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'load-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, LoadError) - end + loaded_module = subject.modules.first + expect(loaded_module.version).to eq('foo') end - it "wraps and re-raises argument errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'argument-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, ArgumentError) - end - end + it 'overrides module loader init if needed' do + # This path doesn't matter so long as it has a Puppetfile within it + path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') + subject = described_class.new(path, {overrides: {environments: {default_branch_override: 'foo'}}}) - it "rejects Puppetfiles with duplicate module names" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'duplicate-module-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error(R10K::Error, /Puppetfiles cannot contain duplicate module names/i) - end - - it "wraps and re-raises name errors" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'name-error') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { - subject.load! - }.to raise_error do |e| - expect_wrapped_error(e, pf_path, NameError) - end + repo = instance_double('R10K::Git::StatefulRepository') + allow(repo).to receive(:resolve).with('bar').and_return(true) + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + + allow(subject.loader).to receive(:puppetfile_content).and_return <<-EOPF + # Track control branch and fall-back to main if no matching branch. + mod 'hieradata', + :git => 'git@git.example.com:organization/hieradata.git', + :branch => :control_branch, + :default_branch => 'main' + EOPF + + expect(subject.logger).to receive(:warn). + with(/Mismatch between passed and initialized.*preferring passed value/) + + subject.load('bar') + loaded_module = subject.modules.first + expect(loaded_module.version).to eq('bar') end - it "accepts a forge module with a version" do + it 'does not warn if passed and initialized default_branch_overrides match' do + # This path doesn't matter so long as it has a Puppetfile within it path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-with-version') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { subject.load! }.not_to raise_error - end - - it "accepts a forge module without a version" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'valid-forge-without-version') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { subject.load! }.not_to raise_error - end - - it "creates a git module and applies the default branch sepcified in the Puppetfile" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - expect { subject.load! }.not_to raise_error - git_module = subject.modules[0] - expect(git_module.default_ref).to eq 'here_lies_the_default_branch' - end - - it "creates a git module and applies the provided default_branch_override" do - path = File.join(PROJECT_ROOT, 'spec', 'fixtures', 'unit', 'puppetfile', 'default-branch-override') - pf_path = File.join(path, 'Puppetfile') - subject = described_class.new(path) - default_branch_override = 'default_branch_override_name' - expect { subject.load!(default_branch_override) }.not_to raise_error - git_module = subject.modules[0] - expect(git_module.default_ref).to eq default_branch_override + subject = described_class.new(path, {overrides: {environments: {default_branch_override: 'foo'}}}) + + repo = instance_double('R10K::Git::StatefulRepository') + allow(repo).to receive(:resolve).with('foo').and_return(true) + allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo) + + allow(subject.loader).to receive(:puppetfile_content).and_return <<-EOPF + # Track control branch and fall-back to main if no matching branch. + mod 'hieradata', + :git => 'git@git.example.com:organization/hieradata.git', + :branch => :control_branch, + :default_branch => 'main' + EOPF + + expect(subject.logger).not_to receive(:warn). + with(/Mismatch between passed and initialized.*preferring passed value/) + + subject.load('foo') + loaded_module = subject.modules.first + expect(loaded_module.version).to eq('foo') end end @@ -294,7 +237,7 @@ subject.accept(visitor) end - it "passes the visitor to each module if the visitor yields" do + it "synchronizes each module if the visitor yields" do visitor = spy('visitor') expect(visitor).to receive(:visit) do |type, other, &block| expect(type).to eq :puppetfile @@ -302,12 +245,12 @@ block.call end - mod1 = spy('module') - expect(mod1).to receive(:accept).with(visitor) - mod2 = spy('module') - expect(mod2).to receive(:accept).with(visitor) + mod1 = instance_double('R10K::Module::Base', :cachedir => :none) + mod2 = instance_double('R10K::Module::Base', :cachedir => :none) + expect(mod1).to receive(:sync) + expect(mod2).to receive(:sync) + expect(subject).to receive(:modules).and_return([mod1, mod2]) - expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]}) subject.accept(visitor) end @@ -323,15 +266,14 @@ block.call end - mod1 = spy('module') - expect(mod1).to receive(:accept).with(visitor) - mod2 = spy('module') - expect(mod2).to receive(:accept).with(visitor) - - expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]}) + mod1 = instance_double('R10K::Module::Base', :cachedir => :none) + mod2 = instance_double('R10K::Module::Base', :cachedir => :none) + expect(mod1).to receive(:sync) + expect(mod2).to receive(:sync) + expect(subject).to receive(:modules).and_return([mod1, mod2]) expect(Thread).to receive(:new).exactly(pool_size).and_call_original - expect(Queue).to receive(:new).and_call_original + expect(Queue).to receive(:new).and_call_original.twice subject.accept(visitor) end @@ -344,22 +286,19 @@ block.call end - mod1 = spy('module1') - mod2 = spy('module2') - mod3 = spy('module3') - mod4 = spy('module4') - mod5 = spy('module5') - mod6 = spy('module6') - - expect(subject).to receive(:modules_by_vcs_cachedir) - .and_return({:none => [mod1, mod2], - "foo-cachedir" => [mod3, mod4], - "bar-cachedir" => [mod5, mod6]}) + m1 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/A') + m2 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/B') + m3 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/C') + m4 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/C') + m5 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/D') + m6 = instance_double('R10K::Module::Base', :cachedir => '/dev/null/D') + + modules = [m1, m2, m3, m4, m5, m6] - queue = subject.modules_queue(visitor) + queue = R10K::ContentSynchronizer.modules_visit_queue(modules, visitor, subject) expect(queue.length).to be 4 queue_array = 4.times.map { queue.pop } - expect(queue_array).to match_array([[mod1], [mod2], [mod3, mod4], [mod5, mod6]]) + expect(queue_array).to match_array([[m1], [m2], [m3, m4], [m5, m6]]) end end end diff -Nru r10k-3.7.0/spec/unit/settings_spec.rb r10k-4.0.0/spec/unit/settings_spec.rb --- r10k-3.7.0/spec/unit/settings_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/settings_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -90,11 +90,50 @@ end end end + + describe "allow_puppetfile_override" do + it 'is false by default' do + expect(subject.evaluate({})[:allow_puppetfile_override]).to eq(false) + end + + it 'can be set to true' do + expect(subject.evaluate({"allow_puppetfile_override" => true})[:allow_puppetfile_override]).to eq(true) + end + + it "raises an error for non-boolean values" do + expect { + subject.evaluate({"allow_puppetfile_override" => 'invalid_string'}) + }.to raise_error do |err| + expect(err.message).to match(/Validation failed for 'forge' settings group/) + expect(err.errors.size).to eq 1 + expect(err.errors[:allow_puppetfile_override]).to be_a_kind_of(ArgumentError) + expect(err.errors[:allow_puppetfile_override].message).to match(/`allow_puppetfile_override` can only be a boolean value, not 'invalid_string'/) + end + end + end end describe "deploy settings" do subject { described_class.deploy_settings } + describe 'exclude_spec' do + it 'is true by default' do + expect(subject.evaluate({})[:exclude_spec]).to eq(true) + end + it 'can be set to false' do + expect(subject.evaluate({"exclude_spec" => false})[:exclude_spec]).to eq(false) + end + it "raises an error for non-boolean values" do + expect { + subject.evaluate({"exclude_spec" => 'invalid_string'}) + }.to raise_error do |err| + expect(err.message).to match(/Validation failed for 'deploy' settings group/) + expect(err.errors.size).to eq 1 + expect(err.errors[:exclude_spec]).to be_a_kind_of(ArgumentError) + expect(err.errors[:exclude_spec].message).to match(/`exclude_spec` can only be a boolean value, not 'invalid_string'/) + end + end + end describe "write_lock" do it "accepts a string with a reason for the write lock" do output = subject.evaluate("write_lock" => "No maintenance window active, code freeze till 2038-01-19") @@ -250,8 +289,14 @@ describe "forge settings" do it "passes settings through to the forge settings" do - output = subject.evaluate("forge" => {"baseurl" => "https://forge.tessier-ashpool.freeside", "proxy" => "https://proxy.tessier-ashpool.freesize:3128"}) - expect(output[:forge]).to eq(:baseurl => "https://forge.tessier-ashpool.freeside", :proxy => "https://proxy.tessier-ashpool.freesize:3128") + output = subject.evaluate("forge" => {"baseurl" => "https://forge.tessier-ashpool.freeside", + "proxy" => "https://proxy.tessier-ashpool.freesize:3128", + "authorization_token" => "faketoken", + "allow_puppetfile_override" => true}) + expect(output[:forge]).to eq(:baseurl => "https://forge.tessier-ashpool.freeside", + :proxy => "https://proxy.tessier-ashpool.freesize:3128", + :authorization_token => "faketoken", + :allow_puppetfile_override => true) end end end diff -Nru r10k-3.7.0/spec/unit/tarball_spec.rb r10k-4.0.0/spec/unit/tarball_spec.rb --- r10k-3.7.0/spec/unit/tarball_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/tarball_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,57 @@ +require 'spec_helper' +require 'r10k/tarball' + +describe R10K::Tarball do + include_context 'Tarball' + + subject { described_class.new('fixture-tarball', fixture_tarball, checksum: fixture_checksum) } + + describe 'initialization' do + it 'initializes' do + expect(subject).to be_kind_of(described_class) + end + end + + describe 'downloading and caching' do + it 'downloads the source to the cache' do + # No cache present initially + expect(File.exist?(subject.cache_path)).to be(false) + expect(subject.cache_valid?).to be(false) + + subject.get + + expect(subject.cache_valid?).to be(true) + expect(File.exist?(subject.cache_path)).to be(true) + end + + let(:raw_content) {[ + './', + './Puppetfile', + './metadata.json', + './spec/', + './environment.conf', + './spec/1', + ]} + + let(:clean_content) {[ + 'Puppetfile', + 'metadata.json', + 'spec', + 'environment.conf', + 'spec/1', + ]} + + it 'returns clean paths when listing cached tarball content' do + iterator = allow(subject).to receive(:each_tarball_entry) + raw_content.each { |entry| iterator.and_yield(entry) } + + expect(subject.paths).to eq(clean_content) + end + end + + describe 'http sources' + + describe 'file sources' + + describe 'syncing' +end diff -Nru r10k-3.7.0/spec/unit/util/cacheable_spec.rb r10k-4.0.0/spec/unit/util/cacheable_spec.rb --- r10k-3.7.0/spec/unit/util/cacheable_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/util/cacheable_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,23 @@ +require 'spec_helper' +require 'r10k/util/cacheable' + +RSpec.describe R10K::Util::Cacheable do + + subject { Object.new.extend(R10K::Util::Cacheable) } + + describe "dirname sanitization" do + let(:input) { 'https://some/git/remote' } + + it 'sanitizes URL to directory name' do + expect(subject.sanitized_dirname(input)).to eq('https---some-git-remote') + end + + context 'with username and password' do + let(:input) { 'https://"user:pa$$w0rd:@authenticated/git/remote' } + + it 'sanitizes authenticated URL to directory name' do + expect(subject.sanitized_dirname(input)).to eq('https---authenticated-git-remote') + end + end + end +end diff -Nru r10k-3.7.0/spec/unit/util/downloader_spec.rb r10k-4.0.0/spec/unit/util/downloader_spec.rb --- r10k-3.7.0/spec/unit/util/downloader_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ r10k-4.0.0/spec/unit/util/downloader_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -0,0 +1,98 @@ +require 'spec_helper' +require 'r10k/util/downloader' + +describe R10K::Util::Downloader do + + subject(:downloader) do + subj = Object.new + subj.extend(R10K::Util::Downloader) + subj.singleton_class.class_eval { public :download } + subj.singleton_class.class_eval { public :http_get } + subj.singleton_class.class_eval { public :file_digest } + subj + end + + let(:tmpdir) { Dir.mktmpdir } + after(:each) { FileUtils.remove_entry_secure(tmpdir) } + + describe 'http_get' do + let(:src_url) { 'https://example.com' } + let(:dst_file) { File.join(tmpdir, 'test.out') } + let(:tarball_uri) { URI('http://tarball.example.com/tarball.tar.gz') } + let(:redirect_uri) { URI('http://redirect.example.com/redirect') } + let(:proxy_uri) { URI('http://user:password@proxy.example.com') } + + it 'downloads a simple file' do + mock_session = instance_double('Net::HTTP', active?: true) + tarball_response = instance_double('Net::HTTPSuccess') + + expect(Net::HTTP).to receive(:new).with(tarball_uri.host, any_args).and_return(mock_session) + expect(Net::HTTPSuccess).to receive(:===).with(tarball_response).and_return(true) + + expect(mock_session).to receive(:request_get).and_yield(tarball_response) + expect(mock_session).to receive(:start).once + expect(mock_session).to receive(:finish).once + + expect { |b| downloader.http_get(tarball_uri, &b) }.to yield_with_args(tarball_response) + end + + it 'follows redirects' do + mock_session_1 = instance_double('Net::HTTP', active?: false) + mock_session_2 = instance_double('Net::HTTP', active?: true) + redirect_response = instance_double('Net::HTTPRedirection') + tarball_response = instance_double('Net::HTTPSuccess') + + expect(Net::HTTP).to receive(:new).with(redirect_uri.host, any_args).and_return(mock_session_1).once + expect(Net::HTTP).to receive(:new).with(tarball_uri.host, any_args).and_return(mock_session_2).once + expect(Net::HTTPRedirection).to receive(:===).with(redirect_response).and_return(true) + expect(Net::HTTPSuccess).to receive(:===).with(tarball_response).and_return(true) + allow(Net::HTTPRedirection).to receive(:===).and_call_original + + expect(mock_session_1).to receive(:request_get).and_yield(redirect_response) + expect(mock_session_2).to receive(:request_get).and_yield(tarball_response) + + # The redirect response should be queried for the redirect location + expect(redirect_response).to receive(:[]).with('location').and_return(tarball_uri.to_s) + + # Both sessions should start and finish cleanly + expect(mock_session_1).to receive(:start).once + expect(mock_session_1).to receive(:finish).once + expect(mock_session_2).to receive(:start).once + expect(mock_session_2).to receive(:finish).once + + expect { |b| downloader.http_get(redirect_uri, &b) }.to yield_with_args(tarball_response) + end + + it 'can use a proxy' do + mock_session = instance_double('Net::HTTP', active?: true) + + expect(Net::HTTP).to receive(:new) + .with(tarball_uri.host, + tarball_uri.port, + proxy_uri.host, + proxy_uri.port, + proxy_uri.user, + proxy_uri.password, + any_args) + .and_return(mock_session) + + expect(mock_session).to receive(:request_get).and_return(:not_yielded) + expect(mock_session).to receive(:start).once + expect(mock_session).to receive(:finish).once + + downloader.http_get(tarball_uri, proxy: proxy_uri) + end + end + + describe 'checksums' do + let(:fixture_checksum) { '0bcea17aa0c5e868c18f0fa042feda770e47c1a4223229f82116ccb3ca33c6e3' } + let(:fixture_tarball) do + File.expand_path('spec/fixtures/integration/git/puppet-boolean-bare.tar', PROJECT_ROOT) + end + + it 'checksums files' do + expect(downloader.file_digest(fixture_tarball)).to eql(fixture_checksum) + end + end +end + diff -Nru r10k-3.7.0/spec/unit/util/purgeable_spec.rb r10k-4.0.0/spec/unit/util/purgeable_spec.rb --- r10k-3.7.0/spec/unit/util/purgeable_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/util/purgeable_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -1,5 +1,6 @@ require 'spec_helper' require 'r10k/util/purgeable' +require 'r10k/util/cleaner' RSpec.describe R10K::Util::Purgeable do let(:managed_directories) do @@ -14,28 +15,25 @@ 'spec/fixtures/unit/util/purgeable/managed_one/expected_1', 'spec/fixtures/unit/util/purgeable/managed_one/new_1', 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1', + 'spec/fixtures/unit/util/purgeable/managed_one/managed_symlink_dir', 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/subdir_expected_1', 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/subdir_new_1', + 'spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/managed_symlink_file', 'spec/fixtures/unit/util/purgeable/managed_two/expected_2', 'spec/fixtures/unit/util/purgeable/managed_two/new_2', + 'spec/fixtures/unit/util/purgeable/managed_two/.hidden', ] end - let(:test_class) do - Struct.new(:managed_directories, :desired_contents) do - include R10K::Util::Purgeable - include R10K::Logging - end - end - - subject { test_class.new(managed_directories, desired_contents) } + subject { R10K::Util::Cleaner.new(managed_directories, desired_contents) } context 'without recurse option' do let(:recurse) { false } describe '#current_contents' do it 'collects direct contents of all managed directories' do - expect(subject.current_contents(recurse)).to contain_exactly(/\/expected_1/, /\/expected_2/, /\/unmanaged_1/, /\/unmanaged_2/, /\/managed_subdir_1/) + expect(subject.current_contents(recurse)).to contain_exactly(/\/expected_1/, /\/expected_2/, /\/unmanaged_1/, /\/unmanaged_2/, + /\/managed_subdir_1/, /\/managed_symlink_dir/, /\/unmanaged_symlink_file/) end end @@ -51,7 +49,7 @@ let(:whitelist) { [] } it 'collects current_contents that should not exist' do - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/) + expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/, /\/unmanaged_symlink_file/) end end @@ -61,7 +59,7 @@ it 'collects current_contents that should not exist except whitelisted items' do expect(subject.logger).to receive(:debug).with(/unmanaged_1.*whitelist match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_2/) + expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_2/, /\/unmanaged_symlink_file/) end end @@ -71,7 +69,7 @@ it 'collects current_contents that should not exist except excluded items' do expect(subject.logger).to receive(:debug2).with(/unmanaged_2.*internal exclusion match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/) + expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_symlink_file/) end end end @@ -104,7 +102,17 @@ describe '#current_contents' do it 'collects contents of all managed directories recursively' do - expect(subject.current_contents(recurse)).to contain_exactly(/\/expected_1/, /\/expected_2/, /\/unmanaged_1/, /\/unmanaged_2/, /\/managed_subdir_1/, /\/subdir_expected_1/, /\/subdir_unmanaged_1/) + expect(subject.current_contents(recurse)). + to contain_exactly(/\/expected_1/, /\/expected_2/, + /\/unmanaged_1/, /\/unmanaged_2/, + /\/managed_symlink_dir/, + /\/unmanaged_symlink_file/, + /\/managed_subdir_1/, + /\/subdir_expected_1/, /\/subdir_unmanaged_1/, + /\/managed_symlink_file/, + /\/unmanaged_symlink_dir/, + /\/subdir_allowlisted_2/, /\/ignored_1/, + /\/\.hidden/) end end @@ -120,7 +128,9 @@ let(:whitelist) { [] } it 'collects current_contents that should not exist recursively' do - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/, /\/subdir_unmanaged_1/) + expect(subject.stale_contents(recurse, exclusions, whitelist)). + to contain_exactly(/\/unmanaged_1/, /\/unmanaged_2/, /\/unmanaged_symlink_file/, /\/subdir_unmanaged_1/, + /\/ignored_1/, /\/subdir_allowlisted_2/, /\/unmanaged_symlink_dir/) end end @@ -130,7 +140,18 @@ it 'collects current_contents that should not exist except whitelisted items' do expect(subject.logger).to receive(:debug).with(/unmanaged_1.*whitelist match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_2/, /\/subdir_unmanaged_1/) + + expect(subject.stale_contents(recurse, exclusions, whitelist)). + to contain_exactly(/\/unmanaged_2/, /\/subdir_unmanaged_1/, /\/unmanaged_symlink_file/, /\/ignored_1/, + /\/subdir_allowlisted_2/, /\/unmanaged_symlink_dir/) + end + + it 'does not collect contents that match recursive globbed whitelist items as intermediate values' do + recursive_whitelist = ['**/managed_subdir_1/**/*'] + expect(subject.logger).not_to receive(:debug).with(/ignored_1/) + + expect(subject.stale_contents(recurse, exclusions, recursive_whitelist)). + to contain_exactly(/\/unmanaged_2/, /\/managed_one\/unmanaged_1/, /\/managed_one\/unmanaged_symlink_file/) end end @@ -140,7 +161,18 @@ it 'collects current_contents that should not exist except excluded items' do expect(subject.logger).to receive(:debug2).with(/unmanaged_2.*internal exclusion match/i) - expect(subject.stale_contents(recurse, exclusions, whitelist)).to contain_exactly(/\/unmanaged_1/, /\/subdir_unmanaged_1/) + + expect(subject.stale_contents(recurse, exclusions, whitelist)). + to contain_exactly(/\/unmanaged_1/, /\/unmanaged_symlink_file/, /\/subdir_unmanaged_1/, /\/ignored_1/, + /\/subdir_allowlisted_2/, /\/unmanaged_symlink_dir/) + end + + it 'does not collect contents that match recursive globbed exclusion items as intermediate values' do + recursive_exclusions = ['**/managed_subdir_1/**/*'] + expect(subject.logger).not_to receive(:debug).with(/ignored_1/) + + expect(subject.stale_contents(recurse, recursive_exclusions, whitelist)). + to contain_exactly(/\/unmanaged_2/, /\/unmanaged_symlink_file/, /\/managed_one\/unmanaged_1/) end end end @@ -177,6 +209,7 @@ it 'does not purge items matching glob at root level' do allow(FileUtils).to receive(:rm_r) expect(FileUtils).to_not receive(:rm_r).with(/\/unmanaged_[12]/, anything) + expect(FileUtils).to_not receive(:rm_r).with(/\/unmanaged_symlink_file/, anything) expect(subject.logger).to receive(:debug).with(/whitelist match/i).at_least(:once) subject.purge!(purge_opts) @@ -185,7 +218,11 @@ end context "recursive whitelist glob" do - let(:whitelist) { managed_directories.collect { |dir| File.join(dir, "**", "*unmanaged*") } } + let(:whitelist) do + managed_directories.flat_map do |dir| + [File.join(dir, "**", "*unmanaged*"), File.join(dir, "**", "subdir_allowlisted_2")] + end + end let(:purge_opts) { { recurse: true, whitelist: whitelist } } describe '#purge!' do @@ -220,7 +257,7 @@ context "when class does not implement #purge_exclusions" do describe '#purge!' do it 'purges normally' do - expect(FileUtils).to receive(:rm_r).at_least(3).times + expect(FileUtils).to receive(:rm_r).at_least(4).times subject.purge!(purge_opts) end diff -Nru r10k-3.7.0/spec/unit/util/setopts_spec.rb r10k-4.0.0/spec/unit/util/setopts_spec.rb --- r10k-3.7.0/spec/unit/util/setopts_spec.rb 2020-11-13 22:27:51.000000000 +0000 +++ r10k-4.0.0/spec/unit/util/setopts_spec.rb 2023-08-01 15:06:51.000000000 +0000 @@ -10,7 +10,10 @@ def initialize(opts = {}) setopts(opts, { - :valid => :self, :alsovalid => :self, :truthyvalid => true, + :valid => :self, + :duplicate => :valid, + :alsovalid => :self, + :truthyvalid => true, :validalias => :valid, :ignoreme => nil }) @@ -53,7 +56,28 @@ }.to raise_error(ArgumentError, /cannot handle option 'notvalid'/) end + it "warns when given an unhandled option and raise_on_unhandled=false" do + test = Class.new { include R10K::Util::Setopts }.new + allow(test).to receive(:logger).and_return(spy) + + test.send(:setopts, {valid: :value, invalid: :value}, + {valid: :self}, + raise_on_unhandled: false) + + expect(test.logger).to have_received(:warn).with(%r{cannot handle option 'invalid'}) + end + it "ignores values that are marked as unhandled" do klass.new(:ignoreme => "IGNORE ME!") end + + it "warns when given conflicting options" do + test = Class.new { include R10K::Util::Setopts }.new + allow(test).to receive(:logger).and_return(spy) + + test.send(:setopts, {valid: :one, duplicate: :two}, + {valid: :arg, duplicate: :arg}) + + expect(test.logger).to have_received(:warn).with(%r{valid.*duplicate.*conflict.*not both}) + end end