diff -Nru ruby-bootsnap-1.4.6/bin/ci ruby-bootsnap-1.9.3/bin/ci --- ruby-bootsnap-1.4.6/bin/ci 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/bin/ci 1970-01-01 00:00:00.000000000 +0000 @@ -1,10 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -if [[ "${MINIMAL_SUPPORT-0}" -eq 1 ]]; then - exec bin/test-minimal-support -else - rake - exec bin/testunit -fi diff -Nru ruby-bootsnap-1.4.6/bootsnap.gemspec ruby-bootsnap-1.9.3/bootsnap.gemspec --- ruby-bootsnap-1.4.6/bootsnap.gemspec 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/bootsnap.gemspec 2021-11-24 16:27:37.000000000 +0000 @@ -20,13 +20,15 @@ 'bug_tracker_uri' => 'https://github.com/Shopify/bootsnap/issues', 'changelog_uri' => 'https://github.com/Shopify/bootsnap/blob/master/CHANGELOG.md', 'source_code_uri' => 'https://github.com/Shopify/bootsnap', + 'allowed_push_host' => 'https://rubygems.org' } - spec.files = %x(git ls-files -z).split("\x0").reject do |f| - f.match(%r{^(test|spec|features)/}) - end + spec.files = %x(git ls-files -z ext lib).split("\x0") + %w(CHANGELOG.md LICENSE.txt README.md) spec.require_paths = %w(lib) + spec.bindir = 'exe' + spec.executables = %w(bootsnap) + spec.required_ruby_version = '>= 2.3.0' if RUBY_PLATFORM =~ /java/ @@ -37,8 +39,8 @@ end spec.add_development_dependency("bundler") - spec.add_development_dependency('rake', '~> 10.0') - spec.add_development_dependency('rake-compiler', '~> 0') + spec.add_development_dependency('rake') + spec.add_development_dependency('rake-compiler') spec.add_development_dependency("minitest", "~> 5.0") spec.add_development_dependency("mocha", "~> 1.2") diff -Nru ruby-bootsnap-1.4.6/CHANGELOG.md ruby-bootsnap-1.9.3/CHANGELOG.md --- ruby-bootsnap-1.4.6/CHANGELOG.md 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/CHANGELOG.md 2021-11-24 16:27:37.000000000 +0000 @@ -1,3 +1,118 @@ +# Unreleased + +# 1.9.3 + +* Only disable the compile cache for source files impacted by [Ruby 3.0.3 [Bug 18250]](https://bugs.ruby-lang.org/issues/18250). + This should keep the performance loss to a minimum. + +# 1.9.2 + +* Disable compile cache if [Ruby 3.0.3's ISeq cache bug](https://bugs.ruby-lang.org/issues/18250) is detected. + AKA `iseq.rb:13 to_binary: wrong argument type false (expected Symbol)` +* Fix `Kernel.load` behavior: before `load 'a'` would load `a.rb` (and other tried extensions) and wouldn't load `a` unless `development_mode: true`, now only `a` would be loaded and files with extensions wouldn't be. + +# 1.9.1 + +* Removed a forgotten debug statement in JSON precompilation. + +# 1.9.0 + +* Added a compilation cache for `JSON.load_file`. (#370) + +# 1.8.1 + +* Fixed support for older Psych. (#369) + +# 1.8.0 + +* Improve support for Pysch 4. (#368) + +# 1.7.7 + +* Fix `require_relative` in evaled code on latest ruby 3.1.0-dev. (#366) + +# 1.7.6 + +* Fix reliance on `set` to be required. +* Fix `Encoding::UndefinedConversionError` error for Rails applications when precompiling cache. (#364) + +# 1.7.5 + +* Handle a regression of Ruby 2.7.3 causing Bootsnap to call the deprecated `untaint` method. (#360) +* Gracefully handle read-only file system as well as other errors preventing to persist the load path cache. (#358) + +# 1.7.4 + +* Stop raising errors when encoutering various file system errors. The cache is now best effort, + if somehow it can't be saved, bootsnapp will gracefully fallback to the original operation (e.g. `Kernel.require`). + (#353, #177, #262) + +# 1.7.3 + +* Disable YAML precompilation when encountering YAML tags. (#351) + +# 1.7.2 + +* Fix compatibility with msgpack < 1. (#349) + +# 1.7.1 + +* Warn Ruby 2.5 users if they turn ISeq caching on. (#327, #244) +* Disable ISeq caching for the whole 2.5.x series again. +* Better handle hashing of Ruby strings. (#318) + +# 1.7.0 + +* Fix detection of YAML files in gems. +* Adds an instrumentation API to monitor cache misses. +* Allow to control the behavior of `require 'bootsnap/setup'` using environment variables. +* Deprecate the `disable_trace` option. +* Deprecate the `ActiveSupport::Dependencies` (AKA Classic autoloader) integration. (#344) + +# 1.6.0 + +* Fix a Ruby 2.7/3.0 issue with `YAML.load_file` keyword arguments. (#342) +* `bootsnap precompile` CLI use multiple processes to complete faster. (#341) +* `bootsnap precompile` CLI also precompile YAML files. (#340) +* Changed the load path cache directory from `$BOOTSNAP_CACHE_DIR/bootsnap-load-path-cache` to `$BOOTSNAP_CACHE_DIR/bootsnap/load-path-cache` for ease of use. (#334) +* Changed the compile cache directory from `$BOOTSNAP_CACHE_DIR/bootsnap-compile-cache` to `$BOOTSNAP_CACHE_DIR/bootsnap/compile-cache` for ease of use. (#334) + +# 1.5.1 + +* Workaround a Ruby bug in InstructionSequence.compile_file. (#332) + +# 1.5.0 + +* Add a command line to statically precompile the ISeq cache. (#326) + +# 1.4.9 + +* [Windows support](https://github.com/Shopify/bootsnap/pull/319) +* [Fix potential crash](https://github.com/Shopify/bootsnap/pull/322) + +# 1.4.8 + +* [Prevent FallbackScan from polluting exception cause](https://github.com/Shopify/bootsnap/pull/314) + +# 1.4.7 + +* Various performance enhancements +* Fix race condition in heavy concurrent load scenarios that would cause bootsnap to raise + +# 1.4.6 + +* Fix bug that was erroneously considering that files containing `.` in the names were being + required if a different file with the same name was already being required + + Example: + + require 'foo' + require 'foo.en' + + Before bootsnap was considering `foo.en` to be the same file as `foo` + +* Use glibc as part of the ruby_platform cache key + # 1.4.5 * MRI 2.7 support diff -Nru ruby-bootsnap-1.4.6/CONTRIBUTING.md ruby-bootsnap-1.9.3/CONTRIBUTING.md --- ruby-bootsnap-1.4.6/CONTRIBUTING.md 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/CONTRIBUTING.md 2021-11-24 16:27:37.000000000 +0000 @@ -19,3 +19,17 @@ 3. Commit your changes (`git commit -am 'Add some feature'`) 4. Push to the branch (`git push origin my-new-feature`) 5. Create a new Pull Request + +## Running Tests on Windows + +### Setup + +1. Ensure you've installed Ruby and the MSYS2 devkit and have ran `ridk enable` in your shell. The `ridk enable` command adds make to the path so the compile rake task works. + +1. Open your shell as Administrator (`Run as Administrator`), as the tests create and delete symlinks + +### Running Tests + +> ridk enable +> bundle install +> bundle exec rake \ No newline at end of file diff -Nru ruby-bootsnap-1.4.6/debian/changelog ruby-bootsnap-1.9.3/debian/changelog --- ruby-bootsnap-1.4.6/debian/changelog 2021-12-03 23:22:27.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/changelog 2021-12-06 04:13:33.000000000 +0000 @@ -1,8 +1,20 @@ -ruby-bootsnap (1.4.6-2build1) jammy; urgency=medium +ruby-bootsnap (1.9.3-1) unstable; urgency=medium - * No-change upload due to ruby3.0 transition, remove ruby2.7 support. + * Team Upload - -- Lucas Kanashiro Fri, 03 Dec 2021 20:22:27 -0300 + [ Daniel Leidert ] + * Fix watch file + + [ Pirate Praveen ] + * Really fix filenamemangle in watch file + * New upstream version 1.9.3 + * Bump Standards-Version to 4.6.0 (no changes needed) + * Refresh patches and drop patches no longer required + * Add ruby-byebug as build dependency + * Update minimum version of ruby-msgpack to 1.4~ + * Update debian/ruby-bootsnap.docs + + -- Pirate Praveen Mon, 06 Dec 2021 09:43:33 +0530 ruby-bootsnap (1.4.6-2) unstable; urgency=medium diff -Nru ruby-bootsnap-1.4.6/debian/control ruby-bootsnap-1.9.3/debian/control --- ruby-bootsnap-1.4.6/debian/control 2021-11-25 11:01:53.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/control 2021-12-06 04:13:33.000000000 +0000 @@ -12,8 +12,9 @@ ruby-minitest (<< 6.0), ruby-minitest, ruby-mocha, - ruby-msgpack -Standards-Version: 4.5.1 + ruby-msgpack (>= 1.4~), + ruby-byebug +Standards-Version: 4.6.0 Vcs-Git: https://salsa.debian.org/ruby-team/ruby-bootsnap.git Vcs-Browser: https://salsa.debian.org/ruby-team/ruby-bootsnap Homepage: https://github.com/Shopify/bootsnap @@ -24,7 +25,7 @@ Package: ruby-bootsnap Architecture: any XB-Ruby-Versions: ${ruby:Versions} -Depends: ruby-msgpack, +Depends: ruby-msgpack (>= 1.4~), ${misc:Depends}, ${shlibs:Depends} Description: Boot large ruby/rails apps faster diff -Nru ruby-bootsnap-1.4.6/debian/patches/no-git-in-gemspec.patch ruby-bootsnap-1.9.3/debian/patches/no-git-in-gemspec.patch --- ruby-bootsnap-1.4.6/debian/patches/no-git-in-gemspec.patch 2021-11-25 11:01:53.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/patches/no-git-in-gemspec.patch 2021-12-06 04:13:33.000000000 +0000 @@ -5,14 +5,12 @@ Last-Update: 2019-06-03 --- a/bootsnap.gemspec +++ b/bootsnap.gemspec -@@ -22,9 +22,7 @@ - 'source_code_uri' => 'https://github.com/Shopify/bootsnap', +@@ -23,7 +23,7 @@ + 'allowed_push_host' => 'https://rubygems.org' } -- spec.files = %x(git ls-files -z).split("\x0").reject do |f| -- f.match(%r{^(test|spec|features)/}) -- end +- spec.files = %x(git ls-files -z ext lib).split("\x0") + %w(CHANGELOG.md LICENSE.txt README.md) + spec.files = Dir.glob('bin/**') + Dir.glob('lib/**') + Dir.glob('ext/**') spec.require_paths = %w(lib) - spec.required_ruby_version = '>= 2.3.0' + spec.bindir = 'exe' diff -Nru ruby-bootsnap-1.4.6/debian/patches/relax-dependencies.patch ruby-bootsnap-1.9.3/debian/patches/relax-dependencies.patch --- ruby-bootsnap-1.4.6/debian/patches/relax-dependencies.patch 2021-11-25 11:01:53.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/patches/relax-dependencies.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -Description: Relax dependencies version -Author: Jongmin Kim -Forwarded: not-needed -Last-Update: 2019-06-03 ---- a/bootsnap.gemspec -+++ b/bootsnap.gemspec -@@ -37,8 +37,8 @@ - end - - spec.add_development_dependency("bundler") -- spec.add_development_dependency('rake', '~> 10.0') -- spec.add_development_dependency('rake-compiler', '~> 0') -+ spec.add_development_dependency('rake', '>= 10.0') -+ spec.add_development_dependency('rake-compiler', '>= 0') - spec.add_development_dependency("minitest", "~> 5.0") - spec.add_development_dependency("mocha", "~> 1.2") - diff -Nru ruby-bootsnap-1.4.6/debian/patches/series ruby-bootsnap-1.9.3/debian/patches/series --- ruby-bootsnap-1.4.6/debian/patches/series 2021-11-25 11:01:53.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/patches/series 2021-12-06 04:13:33.000000000 +0000 @@ -1,3 +1,2 @@ -relax-dependencies.patch no-git-in-gemspec.patch skip-cache-path-tests.patch diff -Nru ruby-bootsnap-1.4.6/debian/patches/skip-cache-path-tests.patch ruby-bootsnap-1.9.3/debian/patches/skip-cache-path-tests.patch --- ruby-bootsnap-1.4.6/debian/patches/skip-cache-path-tests.patch 2021-11-25 11:01:53.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/patches/skip-cache-path-tests.patch 2021-12-06 04:13:33.000000000 +0000 @@ -29,17 +29,17 @@ +=begin def test_no_write_permission_to_cache - path = Help.set_file('a.rb', 'a = 3', 100) - folder = File.dirname(Help.cache_path(@tmp_dir, path)) -@@ -30,6 +31,7 @@ - FileUtils.chmod(0400, folder) - assert_raises(Bootsnap::CompileCache::PermissionError) { load(path) } + if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ + # Always pass this test on Windows because directories aren't read, only +@@ -43,6 +44,7 @@ + load(path) + end end +=end def test_can_open_read_only_cache path = Help.set_file('a.rb', 'a = a = 3', 100) -@@ -98,6 +100,7 @@ +@@ -111,6 +113,7 @@ load(path) end @@ -47,9 +47,9 @@ def test_invalid_cache_file path = Help.set_file('a.rb', 'a = a = 3', 100) cp = Help.cache_path(@tmp_dir, path) -@@ -106,4 +109,5 @@ - load(path) - assert(File.size(cp) > 32) # cache was overwritten +@@ -161,4 +164,5 @@ + ensure + Bootsnap.instrumentation = nil end +=end end diff -Nru ruby-bootsnap-1.4.6/debian/ruby-bootsnap.docs ruby-bootsnap-1.9.3/debian/ruby-bootsnap.docs --- ruby-bootsnap-1.4.6/debian/ruby-bootsnap.docs 2021-11-25 11:01:53.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/ruby-bootsnap.docs 2021-12-06 04:13:33.000000000 +0000 @@ -1,2 +1 @@ -README.jp.md README.md diff -Nru ruby-bootsnap-1.4.6/debian/watch ruby-bootsnap-1.9.3/debian/watch --- ruby-bootsnap-1.4.6/debian/watch 2021-11-25 11:01:53.000000000 +0000 +++ ruby-bootsnap-1.9.3/debian/watch 2021-12-06 04:13:33.000000000 +0000 @@ -1,4 +1,4 @@ version=4 -opts=filenamemangle=s/.+\/v?(\d\S*)\.tar\.gz/ruby-bootsnap-\$1\.tar\.gz/,\ +opts=filenamemangle=s%(?:.*?)?v?(\d[\d.]*)(@ARCHIVE_EXT@)%@PACKAGE@-$1$2%,\ uversionmangle=s/(\d)[_\.\-\+]?(RC|rc|pre|dev|beta|alpha)[.]?(\d*)$/\$1~\$2\$3/ \ https://github.com/Shopify/bootsnap/tags .*/v?(\d\S*)\.tar\.gz diff -Nru ruby-bootsnap-1.4.6/exe/bootsnap ruby-bootsnap-1.9.3/exe/bootsnap --- ruby-bootsnap-1.4.6/exe/bootsnap 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/exe/bootsnap 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'bootsnap/cli' +exit Bootsnap::CLI.new(ARGV).run diff -Nru ruby-bootsnap-1.4.6/ext/bootsnap/bootsnap.c ruby-bootsnap-1.9.3/ext/bootsnap/bootsnap.c --- ruby-bootsnap-1.4.6/ext/bootsnap/bootsnap.c 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/ext/bootsnap/bootsnap.c 2021-11-24 16:27:37.000000000 +0000 @@ -14,6 +14,7 @@ #include "bootsnap.h" #include "ruby.h" #include +#include #include #include #include @@ -32,6 +33,12 @@ #define KEY_SIZE 64 +#define MAX_CREATE_TEMPFILE_ATTEMPT 3 + +#ifndef RB_UNLIKELY + #define RB_UNLIKELY(x) (x) +#endif + /* * An instance of this key is written as the first 64 bytes of each cache file. * The mtime and size members track whether the file contents have changed, and @@ -68,7 +75,7 @@ STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE); /* Effectively a schema version. Bumping invalidates all previous caches */ -static const uint32_t current_version = 2; +static const uint32_t current_version = 3; /* hash of e.g. "x86_64-darwin17", invalidating when ruby is recompiled on a * new OS ABI, etc. */ @@ -86,19 +93,25 @@ static VALUE rb_mBootsnap_CompileCache_Native; static VALUE rb_eBootsnap_CompileCache_Uncompilable; static ID uncompilable; +static ID instrumentation_method; +static VALUE sym_miss; +static VALUE sym_stale; +static bool instrumentation_enabled = false; /* Functions exposed as module functions on Bootsnap::CompileCache::Native */ +static VALUE bs_instrumentation_enabled_set(VALUE self, VALUE enabled); static VALUE bs_compile_option_crc32_set(VALUE self, VALUE crc32_v); -static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler); +static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args); +static VALUE bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler); /* Helpers */ -static uint64_t fnv1a_64(const char *str); -static void bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE]); +static void bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE]); static int bs_read_key(int fd, struct bs_cache_key * key); static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2); -static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler); +static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args); +static VALUE bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler); static int open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance); -static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, const char ** errno_provenance); +static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance); static uint32_t get_ruby_revision(void); static uint32_t get_ruby_platform(void); @@ -106,12 +119,12 @@ * Helper functions to call ruby methods on handler object without crashing on * exception. */ -static int bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data); +static int bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data); static VALUE prot_storage_to_output(VALUE arg); static VALUE prot_input_to_output(VALUE arg); -static void bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag); +static void bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag); static VALUE prot_input_to_storage(VALUE arg); -static int bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data); +static int bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data); struct s2o_data; struct i2o_data; struct i2s_data; @@ -144,15 +157,31 @@ current_ruby_platform = get_ruby_platform(); uncompilable = rb_intern("__bootsnap_uncompilable__"); + instrumentation_method = rb_intern("_instrument"); + + sym_miss = ID2SYM(rb_intern("miss")); + rb_global_variable(&sym_miss); + sym_stale = ID2SYM(rb_intern("stale")); + rb_global_variable(&sym_stale); + + rb_define_module_function(rb_mBootsnap, "instrumentation_enabled=", bs_instrumentation_enabled_set, 1); rb_define_module_function(rb_mBootsnap_CompileCache_Native, "coverage_running?", bs_rb_coverage_running, 0); - rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 3); + rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 4); + rb_define_module_function(rb_mBootsnap_CompileCache_Native, "precompile", bs_rb_precompile, 3); rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1); current_umask = umask(0777); umask(current_umask); } +static VALUE +bs_instrumentation_enabled_set(VALUE self, VALUE enabled) +{ + instrumentation_enabled = RTEST(enabled); + return enabled; +} + /* * Bootsnap's ruby code registers a hook that notifies us via this function * when compile_option changes. These changes invalidate all existing caches. @@ -181,7 +210,7 @@ * - 32 bits doesn't feel collision-resistant enough; 64 is nice. */ static uint64_t -fnv1a_64_iter(uint64_t h, const char *str) +fnv1a_64_iter_cstr(uint64_t h, const char *str) { unsigned char *s = (unsigned char *)str; @@ -194,7 +223,21 @@ } static uint64_t -fnv1a_64(const char *str) +fnv1a_64_iter(uint64_t h, const VALUE str) +{ + unsigned char *s = (unsigned char *)RSTRING_PTR(str); + unsigned char *str_end = (unsigned char *)RSTRING_PTR(str) + RSTRING_LEN(str); + + while (s < str_end) { + h ^= (uint64_t)*s++; + h += (h << 1) + (h << 4) + (h << 5) + (h << 7) + (h << 8) + (h << 40); + } + + return h; +} + +static uint64_t +fnv1a_64(const VALUE str) { uint64_t h = (uint64_t)0xcbf29ce484222325ULL; return fnv1a_64_iter(h, str); @@ -215,7 +258,7 @@ } else { uint64_t hash; - hash = fnv1a_64(StringValueCStr(ruby_revision)); + hash = fnv1a_64(ruby_revision); return (uint32_t)(hash >> 32); } } @@ -235,19 +278,19 @@ VALUE ruby_platform; ruby_platform = rb_const_get(rb_cObject, rb_intern("RUBY_PLATFORM")); - hash = fnv1a_64(RSTRING_PTR(ruby_platform)); + hash = fnv1a_64(ruby_platform); #ifdef _WIN32 return (uint32_t)(hash >> 32) ^ (uint32_t)GetVersion(); #elif defined(__GLIBC__) - hash = fnv1a_64_iter(hash, gnu_get_libc_version()); + hash = fnv1a_64_iter_cstr(hash, gnu_get_libc_version()); return (uint32_t)(hash >> 32); #else struct utsname utsname; /* Not worth crashing if this fails; lose extra cache invalidation potential */ if (uname(&utsname) >= 0) { - hash = fnv1a_64_iter(hash, utsname.version); + hash = fnv1a_64_iter_cstr(hash, utsname.version); } return (uint32_t)(hash >> 32); @@ -262,14 +305,13 @@ * The path will look something like: /12/34567890abcdef */ static void -bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE]) +bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE]) { uint64_t hash = fnv1a_64(path); - uint8_t first_byte = (hash >> (64 - 8)); uint64_t remainder = hash & 0x00ffffffffffffff; - sprintf(*cache_path, "%s/%02x/%014llx", cachedir, first_byte, remainder); + sprintf(*cache_path, "%s/%02"PRIx8"/%014"PRIx64, cachedir, first_byte, remainder); } /* @@ -299,7 +341,7 @@ * conversions on the ruby VALUE arguments before passing them along. */ static VALUE -bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler) +bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args) { FilePathValue(path_v); @@ -315,12 +357,38 @@ char cache_path[MAX_CACHEPATH_SIZE]; /* generate cache path to cache_path */ - bs_cache_path(cachedir, path, &cache_path); + bs_cache_path(cachedir, path_v, &cache_path); - return bs_fetch(path, path_v, cache_path, handler); + return bs_fetch(path, path_v, cache_path, handler, args); } /* + * Entrypoint for Bootsnap::CompileCache::Native.precompile. + * Similar to fetch, but it only generate the cache if missing + * and doesn't return the content. + */ +static VALUE +bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler) +{ + FilePathValue(path_v); + + Check_Type(cachedir_v, T_STRING); + Check_Type(path_v, T_STRING); + + if (RSTRING_LEN(cachedir_v) > MAX_CACHEDIR_SIZE) { + rb_raise(rb_eArgError, "cachedir too long"); + } + + char * cachedir = RSTRING_PTR(cachedir_v); + char * path = RSTRING_PTR(path_v); + char cache_path[MAX_CACHEPATH_SIZE]; + + /* generate cache path to cache_path */ + bs_cache_path(cachedir, path_v, &cache_path); + + return bs_precompile(path, path_v, cache_path, handler); +} +/* * Open the file we want to load/cache and generate a cache key for it if it * was loaded. */ @@ -356,7 +424,8 @@ } #define ERROR_WITH_ERRNO -1 -#define CACHE_MISSING_OR_INVALID -2 +#define CACHE_MISS -2 +#define CACHE_STALE -3 /* * Read the cache key from the given fd, which must have position 0 (e.g. @@ -364,15 +433,16 @@ * * Possible return values: * - 0 (OK, key was loaded) - * - CACHE_MISSING_OR_INVALID (-2) * - ERROR_WITH_ERRNO (-1, errno is set) + * - CACHE_MISS (-2) + * - CACHE_STALE (-3) */ static int bs_read_key(int fd, struct bs_cache_key * key) { ssize_t nread = read(fd, key, KEY_SIZE); if (nread < 0) return ERROR_WITH_ERRNO; - if (nread < KEY_SIZE) return CACHE_MISSING_OR_INVALID; + if (nread < KEY_SIZE) return CACHE_STALE; return 0; } @@ -382,7 +452,8 @@ * * Possible return values: * - 0 (OK, key was loaded) - * - CACHE_MISSING_OR_INVALID (-2) + * - CACHE_MISS (-2) + * - CACHE_STALE (-3) * - ERROR_WITH_ERRNO (-1, errno is set) */ static int @@ -393,8 +464,7 @@ fd = open(path, O_RDONLY); if (fd < 0) { *errno_provenance = "bs_fetch:open_cache_file:open"; - if (errno == ENOENT) return CACHE_MISSING_OR_INVALID; - return ERROR_WITH_ERRNO; + return CACHE_MISS; } #ifdef _WIN32 setmode(fd, O_BINARY); @@ -426,7 +496,7 @@ * or exception, will be the final data returnable to the user. */ static int -fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, const char ** errno_provenance) +fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance) { char * data = NULL; ssize_t nread; @@ -448,13 +518,13 @@ goto done; } if (nread != data_size) { - ret = CACHE_MISSING_OR_INVALID; + ret = CACHE_STALE; goto done; } - storage_data = rb_str_new_static(data, data_size); + storage_data = rb_str_new(data, data_size); - *exception_tag = bs_storage_to_output(handler, storage_data, output_data); + *exception_tag = bs_storage_to_output(handler, args, storage_data, output_data); ret = 0; done: if (data != NULL) xfree(data); @@ -499,25 +569,32 @@ { char template[MAX_CACHEPATH_SIZE + 20]; char * tmp_path; - int fd, ret; + int fd, ret, attempt; ssize_t nwrite; - tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE); - strcat(tmp_path, ".tmp.XXXXXX"); + for (attempt = 0; attempt < MAX_CREATE_TEMPFILE_ATTEMPT; ++attempt) { + tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE); + strcat(tmp_path, ".tmp.XXXXXX"); + + // mkstemp modifies the template to be the actual created path + fd = mkstemp(tmp_path); + if (fd > 0) break; - // mkstemp modifies the template to be the actual created path - fd = mkstemp(tmp_path); - if (fd < 0) { - if (mkpath(tmp_path, 0775) < 0) { + if (attempt == 0 && mkpath(tmp_path, 0775) < 0) { *errno_provenance = "bs_fetch:atomic_write_cache_file:mkpath"; return -1; } - fd = open(tmp_path, O_WRONLY | O_CREAT, 0664); - if (fd < 0) { - *errno_provenance = "bs_fetch:atomic_write_cache_file:open"; - return -1; - } } + if (fd < 0) { + *errno_provenance = "bs_fetch:atomic_write_cache_file:mkstemp"; + return -1; + } + + if (chmod(tmp_path, 0644) < 0) { + *errno_provenance = "bs_fetch:atomic_write_cache_file:chmod"; + return -1; + } + #ifdef _WIN32 setmode(fd, O_BINARY); #endif @@ -615,7 +692,7 @@ * - Return storage_to_output(storage_data) */ static VALUE -bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler) +bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args) { struct bs_cache_key cached_key, current_key; char * contents = NULL; @@ -635,26 +712,34 @@ /* Open the cache key if it exists, and read its cache key in */ cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance); - if (cache_fd == CACHE_MISSING_OR_INVALID) { + if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) { /* This is ok: valid_cache remains false, we re-populate it. */ + if (RB_UNLIKELY(instrumentation_enabled)) { + rb_funcall(rb_mBootsnap, instrumentation_method, 2, cache_fd == CACHE_MISS ? sym_miss : sym_stale, path_v); + } } else if (cache_fd < 0) { goto fail_errno; } else { /* True if the cache existed and no invalidating changes have occurred since * it was generated. */ valid_cache = cache_key_equal(¤t_key, &cached_key); + if (RB_UNLIKELY(instrumentation_enabled)) { + if (!valid_cache) { + rb_funcall(rb_mBootsnap, instrumentation_method, 2, sym_stale, path_v); + } + } } if (valid_cache) { /* Fetch the cache data and return it if we're able to load it successfully */ res = fetch_cached_data( - cache_fd, (ssize_t)cached_key.data_size, handler, + cache_fd, (ssize_t)cached_key.data_size, handler, args, &output_data, &exception_tag, &errno_provenance ); - if (exception_tag != 0) goto raise; - else if (res == CACHE_MISSING_OR_INVALID) valid_cache = 0; - else if (res == ERROR_WITH_ERRNO) goto fail_errno; - else if (!NIL_P(output_data)) goto succeed; /* fast-path, goal */ + if (exception_tag != 0) goto raise; + else if (res == CACHE_MISS || res == CACHE_STALE) valid_cache = 0; + else if (res == ERROR_WITH_ERRNO) goto fail_errno; + else if (!NIL_P(output_data)) goto succeed; /* fast-path, goal */ } close(cache_fd); cache_fd = -1; @@ -662,27 +747,29 @@ /* Read the contents of the source file into a buffer */ if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail_errno; - input_data = rb_str_new_static(contents, current_key.size); + input_data = rb_str_new(contents, current_key.size); /* Try to compile the input_data using input_to_storage(input_data) */ - exception_tag = bs_input_to_storage(handler, input_data, path_v, &storage_data); + exception_tag = bs_input_to_storage(handler, args, input_data, path_v, &storage_data); if (exception_tag != 0) goto raise; /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try * to cache anything; just return input_to_output(input_data) */ if (storage_data == uncompilable) { - bs_input_to_output(handler, input_data, &output_data, &exception_tag); + bs_input_to_output(handler, args, input_data, &output_data, &exception_tag); if (exception_tag != 0) goto raise; goto succeed; } /* If storage_data isn't a string, we can't cache it */ if (!RB_TYPE_P(storage_data, T_STRING)) goto invalid_type_storage_data; - /* Write the cache key and storage_data to the cache directory */ - res = atomic_write_cache_file(cache_path, ¤t_key, storage_data, &errno_provenance); - if (res < 0) goto fail_errno; + /* Attempt to write the cache key and storage_data to the cache directory. + * We do however ignore any failures to persist the cache, as it's better + * to move along, than to interrupt the process. + */ + atomic_write_cache_file(cache_path, ¤t_key, storage_data, &errno_provenance); /* Having written the cache, now convert storage_data to output_data */ - exception_tag = bs_storage_to_output(handler, storage_data, &output_data); + exception_tag = bs_storage_to_output(handler, args, storage_data, &output_data); if (exception_tag != 0) goto raise; /* If output_data is nil, delete the cache entry and generate the output @@ -692,7 +779,7 @@ errno_provenance = "bs_fetch:unlink"; goto fail_errno; } - bs_input_to_output(handler, input_data, &output_data, &exception_tag); + bs_input_to_output(handler, args, input_data, &output_data, &exception_tag); if (exception_tag != 0) goto raise; } @@ -723,6 +810,79 @@ #undef CLEANUP } +static VALUE +bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler) +{ + struct bs_cache_key cached_key, current_key; + char * contents = NULL; + int cache_fd = -1, current_fd = -1; + int res, valid_cache = 0, exception_tag = 0; + const char * errno_provenance = NULL; + + VALUE input_data; /* data read from source file, e.g. YAML or ruby source */ + VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */ + + /* Open the source file and generate a cache key for it */ + current_fd = open_current_file(path, ¤t_key, &errno_provenance); + if (current_fd < 0) goto fail; + + /* Open the cache key if it exists, and read its cache key in */ + cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance); + if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) { + /* This is ok: valid_cache remains false, we re-populate it. */ + } else if (cache_fd < 0) { + goto fail; + } else { + /* True if the cache existed and no invalidating changes have occurred since + * it was generated. */ + valid_cache = cache_key_equal(¤t_key, &cached_key); + } + + if (valid_cache) { + goto succeed; + } + + close(cache_fd); + cache_fd = -1; + /* Cache is stale, invalid, or missing. Regenerate and write it out. */ + + /* Read the contents of the source file into a buffer */ + if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail; + input_data = rb_str_new(contents, current_key.size); + + /* Try to compile the input_data using input_to_storage(input_data) */ + exception_tag = bs_input_to_storage(handler, Qnil, input_data, path_v, &storage_data); + if (exception_tag != 0) goto fail; + + /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try + * to cache anything; just return false */ + if (storage_data == uncompilable) { + goto fail; + } + /* If storage_data isn't a string, we can't cache it */ + if (!RB_TYPE_P(storage_data, T_STRING)) goto fail; + + /* Write the cache key and storage_data to the cache directory */ + res = atomic_write_cache_file(cache_path, ¤t_key, storage_data, &errno_provenance); + if (res < 0) goto fail; + + goto succeed; + +#define CLEANUP \ + if (contents != NULL) xfree(contents); \ + if (current_fd >= 0) close(current_fd); \ + if (cache_fd >= 0) close(cache_fd); + +succeed: + CLEANUP; + return Qtrue; +fail: + CLEANUP; + return Qfalse; +#undef CLEANUP +} + + /*****************************************************************************/ /********************* Handler Wrappers **************************************/ /***************************************************************************** @@ -742,11 +902,13 @@ struct s2o_data { VALUE handler; + VALUE args; VALUE storage_data; }; struct i2o_data { VALUE handler; + VALUE args; VALUE input_data; }; @@ -760,15 +922,16 @@ prot_storage_to_output(VALUE arg) { struct s2o_data * data = (struct s2o_data *)arg; - return rb_funcall(data->handler, rb_intern("storage_to_output"), 1, data->storage_data); + return rb_funcall(data->handler, rb_intern("storage_to_output"), 2, data->storage_data, data->args); } static int -bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data) +bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data) { int state; struct s2o_data s2o_data = { .handler = handler, + .args = args, .storage_data = storage_data, }; *output_data = rb_protect(prot_storage_to_output, (VALUE)&s2o_data, &state); @@ -776,10 +939,11 @@ } static void -bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag) +bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag) { struct i2o_data i2o_data = { .handler = handler, + .args = args, .input_data = input_data, }; *output_data = rb_protect(prot_input_to_output, (VALUE)&i2o_data, exception_tag); @@ -789,7 +953,7 @@ prot_input_to_output(VALUE arg) { struct i2o_data * data = (struct i2o_data *)arg; - return rb_funcall(data->handler, rb_intern("input_to_output"), 1, data->input_data); + return rb_funcall(data->handler, rb_intern("input_to_output"), 2, data->input_data, data->args); } static VALUE @@ -800,7 +964,7 @@ } static VALUE -rescue_input_to_storage(VALUE arg) +rescue_input_to_storage(VALUE arg, VALUE e) { return uncompilable; } @@ -816,7 +980,7 @@ } static int -bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data) +bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data) { int state; struct i2s_data i2s_data = { diff -Nru ruby-bootsnap-1.4.6/ext/bootsnap/extconf.rb ruby-bootsnap-1.9.3/ext/bootsnap/extconf.rb --- ruby-bootsnap-1.4.6/ext/bootsnap/extconf.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/ext/bootsnap/extconf.rb 2021-11-24 16:27:37.000000000 +0000 @@ -1,19 +1,24 @@ # frozen_string_literal: true require("mkmf") -$CFLAGS << ' -O3 ' -$CFLAGS << ' -std=c99' -# ruby.h has some -Wpedantic fails in some cases -# (e.g. https://github.com/Shopify/bootsnap/issues/15) -unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC']) - $CFLAGS << ' -Wall' - $CFLAGS << ' -Werror' - $CFLAGS << ' -Wextra' - $CFLAGS << ' -Wpedantic' +if RUBY_ENGINE == 'ruby' + $CFLAGS << ' -O3 ' + $CFLAGS << ' -std=c99' - $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is. - $CFLAGS << ' -Wno-keyword-macro' # hiding return - $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno -end + # ruby.h has some -Wpedantic fails in some cases + # (e.g. https://github.com/Shopify/bootsnap/issues/15) + unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC']) + $CFLAGS << ' -Wall' + $CFLAGS << ' -Werror' + $CFLAGS << ' -Wextra' + $CFLAGS << ' -Wpedantic' + + $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is. + $CFLAGS << ' -Wno-keyword-macro' # hiding return + $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno + end -create_makefile("bootsnap/bootsnap") + create_makefile("bootsnap/bootsnap") +else + File.write("Makefile", dummy_makefile($srcdir).join("")) +end diff -Nru ruby-bootsnap-1.4.6/Gemfile ruby-bootsnap-1.9.3/Gemfile --- ruby-bootsnap-1.4.6/Gemfile 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/Gemfile 2021-11-24 16:27:37.000000000 +0000 @@ -4,6 +4,11 @@ # Specify your gem's dependencies in bootsnap.gemspec gemspec +if ENV["PSYCH_4"] + gem "psych", ">= 4" +end + group :development do gem 'rubocop' + gem 'byebug', platform: :ruby end diff -Nru ruby-bootsnap-1.4.6/.github/CODEOWNERS ruby-bootsnap-1.9.3/.github/CODEOWNERS --- ruby-bootsnap-1.4.6/.github/CODEOWNERS 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/.github/CODEOWNERS 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -# mvm:maintainer -* @burke diff -Nru ruby-bootsnap-1.4.6/.github/issue_template.md ruby-bootsnap-1.9.3/.github/issue_template.md --- ruby-bootsnap-1.4.6/.github/issue_template.md 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/.github/issue_template.md 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,22 @@ +### I made sure the issue is in bootsnap + + + +### Steps to reproduce + + +### Expected behavior + + +### Actual behavior + + +### System configuration + +**Bootsnap version**: + +**Ruby version**: + +**Rails version**: diff -Nru ruby-bootsnap-1.4.6/.github/workflows/ci.yaml ruby-bootsnap-1.9.3/.github/workflows/ci.yaml --- ruby-bootsnap-1.4.6/.github/workflows/ci.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/.github/workflows/ci.yaml 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,70 @@ +name: ci + +on: + pull_request: + branches: + - master + push: + branches: + - master + schedule: + - cron: '45 4 * * *' + +jobs: + platforms: + strategy: + matrix: + os: [ubuntu, macos, windows] + ruby: ['2.5'] + runs-on: ${{ matrix.os }}-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + - run: bundle install + - run: bundle exec rake + rubies: + strategy: + matrix: + os: [ubuntu] + ruby: ['2.3', '2.4', '2.5', '2.6', '2.7', '3.0', 'ruby-head', 'debug'] + runs-on: ${{ matrix.os }}-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + - run: bundle install + - run: bundle exec rake + + psych4: + strategy: + matrix: + os: [ubuntu] + ruby: ['3.0'] + runs-on: ${{ matrix.os }}-latest + env: + PSYCH_4: "1" + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + - run: bundle install + - run: bundle exec rake + + + minimal: + strategy: + matrix: + os: [ubuntu] + ruby: ['jruby', 'truffleruby'] + runs-on: ${{ matrix.os }}-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + - run: bundle install + - run: bin/test-minimal-support diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/cli/worker_pool.rb ruby-bootsnap-1.9.3/lib/bootsnap/cli/worker_pool.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/cli/worker_pool.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/cli/worker_pool.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +module Bootsnap + class CLI + class WorkerPool + class << self + def create(size:, jobs:) + if size > 0 && Process.respond_to?(:fork) + new(size: size, jobs: jobs) + else + Inline.new(jobs: jobs) + end + end + end + + class Inline + def initialize(jobs: {}) + @jobs = jobs + end + + def push(job, *args) + @jobs.fetch(job).call(*args) + nil + end + + def spawn + # noop + end + + def shutdown + # noop + end + end + + class Worker + attr_reader :to_io, :pid + + def initialize(jobs) + @jobs = jobs + @pipe_out, @to_io = IO.pipe(binmode: true) + # Set the writer encoding to binary since IO.pipe only sets it for the reader. + # https://github.com/rails/rails/issues/16514#issuecomment-52313290 + @to_io.set_encoding(Encoding::BINARY) + + @pid = nil + end + + def write(message, block: true) + payload = Marshal.dump(message) + if block + to_io.write(payload) + true + else + to_io.write_nonblock(payload, exception: false) != :wait_writable + end + end + + def close + to_io.close + end + + def work_loop + loop do + job, *args = Marshal.load(@pipe_out) + return if job == :exit + @jobs.fetch(job).call(*args) + end + rescue IOError + nil + end + + def spawn + @pid = Process.fork do + to_io.close + work_loop + exit!(0) + end + @pipe_out.close + true + end + end + + def initialize(size:, jobs: {}) + @size = size + @jobs = jobs + @queue = Queue.new + @pids = [] + end + + def spawn + @workers = @size.times.map { Worker.new(@jobs) } + @workers.each(&:spawn) + @dispatcher_thread = Thread.new { dispatch_loop } + @dispatcher_thread.abort_on_exception = true + true + end + + def dispatch_loop + loop do + case job = @queue.pop + when nil + @workers.each do |worker| + worker.write([:exit]) + worker.close + end + return true + else + unless @workers.sample.write(job, block: false) + free_worker.write(job) + end + end + end + end + + def free_worker + IO.select(nil, @workers)[1].sample + end + + def push(*args) + @queue.push(args) + nil + end + + def shutdown + @queue.close + @dispatcher_thread.join + @workers.each do |worker| + _pid, status = Process.wait2(worker.pid) + return status.exitstatus unless status.success? + end + nil + end + end + end +end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/cli.rb ruby-bootsnap-1.9.3/lib/bootsnap/cli.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/cli.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/cli.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,281 @@ +# frozen_string_literal: true + +require 'bootsnap' +require 'bootsnap/cli/worker_pool' +require 'optparse' +require 'fileutils' +require 'etc' + +module Bootsnap + class CLI + unless Regexp.method_defined?(:match?) + module RegexpMatchBackport + refine Regexp do + def match?(string) + !!match(string) + end + end + end + using RegexpMatchBackport + end + + attr_reader :cache_dir, :argv + + attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs + + def initialize(argv) + @argv = argv + self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache') + self.compile_gemfile = false + self.exclude = nil + self.verbose = false + self.jobs = Etc.nprocessors + self.iseq = true + self.yaml = true + self.json = true + end + + def precompile_command(*sources) + require 'bootsnap/compile_cache/iseq' + require 'bootsnap/compile_cache/yaml' + require 'bootsnap/compile_cache/json' + + fix_default_encoding do + Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir + Bootsnap::CompileCache::YAML.init! + Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir + Bootsnap::CompileCache::JSON.init! + Bootsnap::CompileCache::JSON.cache_dir = self.cache_dir + + @work_pool = WorkerPool.create(size: jobs, jobs: { + ruby: method(:precompile_ruby), + yaml: method(:precompile_yaml), + json: method(:precompile_json), + }) + @work_pool.spawn + + main_sources = sources.map { |d| File.expand_path(d) } + precompile_ruby_files(main_sources) + precompile_yaml_files(main_sources) + precompile_json_files(main_sources) + + if compile_gemfile + # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling. + gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact) + precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude) + + # Gems that include JSON or YAML files usually don't put them in `lib/`. + # So we look at the gem root. + gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+} + gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq + precompile_yaml_files(gem_paths, exclude: gem_exclude) + precompile_json_files(gem_paths, exclude: gem_exclude) + end + + if exitstatus = @work_pool.shutdown + exit(exitstatus) + end + end + 0 + end + + dir_sort = begin + Dir[__FILE__, sort: false] + true + rescue ArgumentError, TypeError + false + end + + if dir_sort + def list_files(path, pattern) + if File.directory?(path) + Dir[File.join(path, pattern), sort: false] + elsif File.exist?(path) + [path] + else + [] + end + end + else + def list_files(path, pattern) + if File.directory?(path) + Dir[File.join(path, pattern)] + elsif File.exist?(path) + [path] + else + [] + end + end + end + + def run + parser.parse!(argv) + command = argv.shift + method = "#{command}_command" + if respond_to?(method) + public_send(method, *argv) + else + invalid_usage!("Unknown command: #{command}") + end + end + + private + + def precompile_yaml_files(load_paths, exclude: self.exclude) + return unless yaml + + load_paths.each do |path| + if !exclude || !exclude.match?(path) + list_files(path, '**/*.{yml,yaml}').each do |yaml_file| + # We ignore hidden files to not match the various .ci.yml files + if !File.basename(yaml_file).start_with?('.') && (!exclude || !exclude.match?(yaml_file)) + @work_pool.push(:yaml, yaml_file) + end + end + end + end + end + + def precompile_yaml(*yaml_files) + Array(yaml_files).each do |yaml_file| + if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir) + STDERR.puts(yaml_file) if verbose + end + end + end + + def precompile_json_files(load_paths, exclude: self.exclude) + return unless json + + load_paths.each do |path| + if !exclude || !exclude.match?(path) + list_files(path, '**/*.json').each do |json_file| + # We ignore hidden files to not match the various .config.json files + if !File.basename(json_file).start_with?('.') && (!exclude || !exclude.match?(json_file)) + @work_pool.push(:json, json_file) + end + end + end + end + end + + def precompile_json(*json_files) + Array(json_files).each do |json_file| + if CompileCache::JSON.precompile(json_file, cache_dir: cache_dir) + STDERR.puts(json_file) if verbose + end + end + end + + def precompile_ruby_files(load_paths, exclude: self.exclude) + return unless iseq + + load_paths.each do |path| + if !exclude || !exclude.match?(path) + list_files(path, '**/*.rb').each do |ruby_file| + if !exclude || !exclude.match?(ruby_file) + @work_pool.push(:ruby, ruby_file) + end + end + end + end + end + + def precompile_ruby(*ruby_files) + Array(ruby_files).each do |ruby_file| + if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir) + STDERR.puts(ruby_file) if verbose + end + end + end + + def fix_default_encoding + if Encoding.default_external == Encoding::US_ASCII + Encoding.default_external = Encoding::UTF_8 + begin + yield + ensure + Encoding.default_external = Encoding::US_ASCII + end + else + yield + end + end + + def invalid_usage!(message) + STDERR.puts message + STDERR.puts + STDERR.puts parser + 1 + end + + def cache_dir=(dir) + @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache')) + end + + def exclude_pattern(pattern) + (@exclude_patterns ||= []) << Regexp.new(pattern) + self.exclude = Regexp.union(@exclude_patterns) + end + + def parser + @parser ||= OptionParser.new do |opts| + opts.banner = "Usage: bootsnap COMMAND [ARGS]" + opts.separator "" + opts.separator "GLOBAL OPTIONS" + opts.separator "" + + help = <<~EOS + Path to the bootsnap cache directory. Defaults to tmp/cache + EOS + opts.on('--cache-dir DIR', help.strip) do |dir| + self.cache_dir = dir + end + + help = <<~EOS + Print precompiled paths. + EOS + opts.on('--verbose', '-v', help.strip) do + self.verbose = true + end + + help = <<~EOS + Number of workers to use. Default to number of processors, set to 0 to disable multi-processing. + EOS + opts.on('--jobs JOBS', '-j', help.strip) do |jobs| + self.jobs = Integer(jobs) + end + + opts.separator "" + opts.separator "COMMANDS" + opts.separator "" + opts.separator " precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories" + + help = <<~EOS + Precompile the gems in Gemfile + EOS + opts.on('--gemfile', help) { self.compile_gemfile = true } + + help = <<~EOS + Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api' + EOS + opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) } + + help = <<~EOS + Disable ISeq (.rb) precompilation. + EOS + opts.on('--no-iseq', help) { self.iseq = false } + + help = <<~EOS + Disable YAML precompilation. + EOS + opts.on('--no-yaml', help) { self.yaml = false } + + help = <<~EOS + Disable JSON precompilation. + EOS + opts.on('--no-json', help) { self.json = false } + end + end + end +end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache/iseq.rb ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache/iseq.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache/iseq.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache/iseq.rb 2021-11-24 16:27:37.000000000 +0000 @@ -9,13 +9,38 @@ attr_accessor(:cache_dir) end - def self.input_to_storage(_, path) - RubyVM::InstructionSequence.compile_file(path).to_binary - rescue SyntaxError - raise(Uncompilable, 'syntax error') + has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250 + if defined? RubyVM::InstructionSequence + RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end; def foo(**); ->{ super }; end").to_binary + end + false + rescue TypeError + true + end + + if has_ruby_bug_18250 + def self.input_to_storage(_, path) + iseq = begin + RubyVM::InstructionSequence.compile_file(path) + rescue SyntaxError + raise(Uncompilable, 'syntax error') + end + + begin + iseq.to_binary + rescue TypeError + raise(Uncompilable, 'ruby bug #18250') + end + end + else + def self.input_to_storage(_, path) + RubyVM::InstructionSequence.compile_file(path).to_binary + rescue SyntaxError + raise(Uncompilable, 'syntax error') + end end - def self.storage_to_output(binary) + def self.storage_to_output(binary, _args) RubyVM::InstructionSequence.load_from_binary(binary) rescue RuntimeError => e if e.message == 'broken binary format' @@ -26,7 +51,24 @@ end end - def self.input_to_output(_) + def self.fetch(path, cache_dir: ISeq.cache_dir) + Bootsnap::CompileCache::Native.fetch( + cache_dir, + path.to_s, + Bootsnap::CompileCache::ISeq, + nil, + ) + end + + def self.precompile(path, cache_dir: ISeq.cache_dir) + Bootsnap::CompileCache::Native.precompile( + cache_dir, + path.to_s, + Bootsnap::CompileCache::ISeq, + ) + end + + def self.input_to_output(_data, _kwargs) nil # ruby handles this end @@ -35,11 +77,7 @@ # Having coverage enabled prevents iseq dumping/loading. return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running? - Bootsnap::CompileCache::Native.fetch( - Bootsnap::CompileCache::ISeq.cache_dir, - path.to_s, - Bootsnap::CompileCache::ISeq - ) + Bootsnap::CompileCache::ISeq.fetch(path.to_s) rescue Errno::EACCES Bootsnap::CompileCache.permission_error(path) rescue RuntimeError => e @@ -60,6 +98,7 @@ crc = Zlib.crc32(option.inspect) Bootsnap::CompileCache::Native.compile_option_crc32 = crc end + compile_option_updated def self.install!(cache_dir) Bootsnap::CompileCache::ISeq.cache_dir = cache_dir diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache/json.rb ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache/json.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache/json.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache/json.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,79 @@ +# frozen_string_literal: true +require('bootsnap/bootsnap') + +module Bootsnap + module CompileCache + module JSON + class << self + attr_accessor(:msgpack_factory, :cache_dir, :supported_options) + + def input_to_storage(payload, _) + obj = ::JSON.parse(payload) + msgpack_factory.dump(obj) + end + + def storage_to_output(data, kwargs) + if kwargs && kwargs.key?(:symbolize_names) + kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names) + end + msgpack_factory.load(data, kwargs) + end + + def input_to_output(data, kwargs) + ::JSON.parse(data, **(kwargs || {})) + end + + def precompile(path, cache_dir: self.cache_dir) + Bootsnap::CompileCache::Native.precompile( + cache_dir, + path.to_s, + self, + ) + end + + def install!(cache_dir) + self.cache_dir = cache_dir + init! + if ::JSON.respond_to?(:load_file) + ::JSON.singleton_class.prepend(Patch) + end + end + + def init! + require('json') + require('msgpack') + + self.msgpack_factory = MessagePack::Factory.new + self.supported_options = [:symbolize_names] + if ::JSON.parse('["foo"]', freeze: true).first.frozen? + self.supported_options = [:freeze] + end + self.supported_options.freeze + end + end + + module Patch + def load_file(path, *args) + return super if args.size > 1 + if kwargs = args.first + return super unless kwargs.is_a?(Hash) + return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty? + end + + begin + ::Bootsnap::CompileCache::Native.fetch( + Bootsnap::CompileCache::JSON.cache_dir, + File.realpath(path), + ::Bootsnap::CompileCache::JSON, + kwargs, + ) + rescue Errno::EACCES + ::Bootsnap::CompileCache.permission_error(path) + end + end + + ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true) + end + end + end +end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache/yaml.rb ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache/yaml.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache/yaml.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache/yaml.rb 2021-11-24 16:27:37.000000000 +0000 @@ -5,58 +5,159 @@ module CompileCache module YAML class << self - attr_accessor(:msgpack_factory) - end + attr_accessor(:msgpack_factory, :cache_dir, :supported_options) - def self.input_to_storage(contents, _) - raise(Uncompilable) if contents.index("!ruby/object") - obj = ::YAML.load(contents) - msgpack_factory.packer.write(obj).to_s - rescue NoMethodError, RangeError - # if the object included things that we can't serialize, fall back to - # Marshal. It's a bit slower, but can encode anything yaml can. - # NoMethodError is unexpected types; RangeError is Bignums - Marshal.dump(obj) - end + def input_to_storage(contents, _) + obj = strict_load(contents) + msgpack_factory.dump(obj) + rescue NoMethodError, RangeError + # The object included things that we can't serialize + raise(Uncompilable) + end - def self.storage_to_output(data) - # This could have a meaning in messagepack, and we're being a little lazy - # about it. -- but a leading 0x04 would indicate the contents of the YAML - # is a positive integer, which is rare, to say the least. - if data[0] == 0x04.chr && data[1] == 0x08.chr - Marshal.load(data) - else - msgpack_factory.unpacker.feed(data).read + def storage_to_output(data, kwargs) + if kwargs && kwargs.key?(:symbolize_names) + kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names) + end + msgpack_factory.load(data, kwargs) end - end - def self.input_to_output(data) - ::YAML.load(data) + def input_to_output(data, kwargs) + if ::YAML.respond_to?(:unsafe_load) + ::YAML.unsafe_load(data, **(kwargs || {})) + else + ::YAML.load(data, **(kwargs || {})) + end + end + + def strict_load(payload, *args) + ast = ::YAML.parse(payload) + return ast unless ast + strict_visitor.create(*args).visit(ast) + end + ruby2_keywords :strict_load if respond_to?(:ruby2_keywords, true) + + def precompile(path, cache_dir: YAML.cache_dir) + Bootsnap::CompileCache::Native.precompile( + cache_dir, + path.to_s, + Bootsnap::CompileCache::YAML, + ) + end + + def install!(cache_dir) + self.cache_dir = cache_dir + init! + ::YAML.singleton_class.prepend(Patch) + end + + def init! + require('yaml') + require('msgpack') + require('date') + + if Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file) + Patch.send(:remove_method, :unsafe_load_file) + end + if Patch.method_defined?(:load_file) && ::YAML::VERSION >= '4' + Patch.send(:remove_method, :load_file) + end + + # MessagePack serializes symbols as strings by default. + # We want them to roundtrip cleanly, so we use a custom factory. + # see: https://github.com/msgpack/msgpack-ruby/pull/122 + factory = MessagePack::Factory.new + factory.register_type(0x00, Symbol) + + if defined? MessagePack::Timestamp + factory.register_type( + MessagePack::Timestamp::TYPE, # or just -1 + Time, + packer: MessagePack::Time::Packer, + unpacker: MessagePack::Time::Unpacker + ) + + marshal_fallback = { + packer: ->(value) { Marshal.dump(value) }, + unpacker: ->(payload) { Marshal.load(payload) }, + } + { + Date => 0x01, + Regexp => 0x02, + }.each do |type, code| + factory.register_type(code, type, marshal_fallback) + end + end + + self.msgpack_factory = factory + + self.supported_options = [] + params = ::YAML.method(:load).parameters + if params.include?([:key, :symbolize_names]) + self.supported_options << :symbolize_names + end + if params.include?([:key, :freeze]) + if factory.load(factory.dump('yaml'), freeze: true).frozen? + self.supported_options << :freeze + end + end + self.supported_options.freeze + end + + def strict_visitor + self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do + def visit(target) + if target.tag + raise Uncompilable, "YAML tags are not supported: #{target.tag}" + end + super + end + end + end end - def self.install!(cache_dir) - require('yaml') - require('msgpack') - - # MessagePack serializes symbols as strings by default. - # We want them to roundtrip cleanly, so we use a custom factory. - # see: https://github.com/msgpack/msgpack-ruby/pull/122 - factory = MessagePack::Factory.new - factory.register_type(0x00, Symbol) - Bootsnap::CompileCache::YAML.msgpack_factory = factory + module Patch + def load_file(path, *args) + return super if args.size > 1 + if kwargs = args.first + return super unless kwargs.is_a?(Hash) + return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty? + end + + begin + ::Bootsnap::CompileCache::Native.fetch( + Bootsnap::CompileCache::YAML.cache_dir, + File.realpath(path), + ::Bootsnap::CompileCache::YAML, + kwargs, + ) + rescue Errno::EACCES + ::Bootsnap::CompileCache.permission_error(path) + end + end + + ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true) + + def unsafe_load_file(path, *args) + return super if args.size > 1 + if kwargs = args.first + return super unless kwargs.is_a?(Hash) + return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty? + end - klass = class << ::YAML; self; end - klass.send(:define_method, :load_file) do |path| begin - Bootsnap::CompileCache::Native.fetch( - cache_dir, - path, - Bootsnap::CompileCache::YAML + ::Bootsnap::CompileCache::Native.fetch( + Bootsnap::CompileCache::YAML.cache_dir, + File.realpath(path), + ::Bootsnap::CompileCache::YAML, + kwargs, ) rescue Errno::EACCES - Bootsnap::CompileCache.permission_error(path) + ::Bootsnap::CompileCache.permission_error(path) end end + + ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true) end end end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache.rb ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/compile_cache.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/compile_cache.rb 2021-11-24 16:27:37.000000000 +0000 @@ -4,7 +4,7 @@ Error = Class.new(StandardError) PermissionError = Class.new(Error) - def self.setup(cache_dir:, iseq:, yaml:) + def self.setup(cache_dir:, iseq:, yaml:, json:) if iseq if supported? require_relative('compile_cache/iseq') @@ -22,6 +22,15 @@ warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby") end end + + if json + if supported? + require_relative('compile_cache/json') + Bootsnap::CompileCache::JSON.install!(cache_dir) + elsif $VERBOSE + warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby") + end + end end def self.permission_error(path) @@ -34,9 +43,9 @@ end def self.supported? - # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), and >= 2.3.0 + # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0 RUBY_ENGINE == 'ruby' && - RUBY_PLATFORM =~ /darwin|linux|bsd/ && + RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/ && Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0") end end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/cache.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/cache.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/cache.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/cache.rb 2021-11-24 16:27:37.000000000 +0000 @@ -10,8 +10,8 @@ def initialize(store, path_obj, development_mode: false) @development_mode = development_mode @store = store - @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped. - @path_obj = path_obj.map! { |f| File.exist?(f) ? File.realpath(f) : f } + @mutex = Mutex.new + @path_obj = path_obj.map! { |f| PathScanner.os_path(File.exist?(f) ? File.realpath(f) : f.dup) } @has_relative_paths = nil reinitialize end @@ -44,14 +44,20 @@ # Try to resolve this feature to an absolute path without traversing the # loadpath. - def find(feature) + def find(feature, try_extensions: true) reinitialize if (@has_relative_paths && dir_changed?) || stale? - feature = feature.to_s + feature = feature.to_s.freeze + return feature if absolute_path?(feature) - return expand_path(feature) if feature.start_with?('./') + + if feature.start_with?('./', '../') + return try_extensions ? expand_path(feature) : File.expand_path(feature).freeze + end + @mutex.synchronize do - x = search_index(feature) + x = search_index(feature, try_extensions: try_extensions) return x if x + return unless try_extensions # Ruby has some built-in features that require lies about. # For example, 'enumerator' is built in. If you require it, ruby @@ -177,16 +183,24 @@ end if DLEXT2 - def search_index(f) - try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f) + def search_index(f, try_extensions: true) + if try_extensions + try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f) + else + try_index(f) + end end def maybe_append_extension(f) try_ext(f + DOT_RB) || try_ext(f + DLEXT) || try_ext(f + DLEXT2) || f end else - def search_index(f) - try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f) + def search_index(f, try_extensions: true) + if try_extensions + try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f) + else + try_index(f) + end end def maybe_append_extension(f) @@ -194,9 +208,26 @@ end end - def try_index(f) - if (p = @index[f]) - p + '/' + f + s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze + if s.respond_to?(:-@) + if (-s).equal?(s) && (-s.dup).equal?(s) || RUBY_VERSION >= '2.7' + def try_index(f) + if (p = @index[f]) + -(File.join(p, f).freeze) + end + end + else + def try_index(f) + if (p = @index[f]) + -File.join(p, f).untaint + end + end + end + else + def try_index(f) + if (p = @index[f]) + File.join(p, f) + end end end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/change_observer.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/change_observer.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/change_observer.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/change_observer.rb 2021-11-24 16:27:37.000000000 +0000 @@ -15,18 +15,20 @@ @lpc_observer.push_paths(self, *entries.map(&:to_s)) super end + alias_method :append, :push def unshift(*entries) @lpc_observer.unshift_paths(self, *entries.map(&:to_s)) super end + alias_method :prepend, :unshift def concat(entries) @lpc_observer.push_paths(self, *entries.map(&:to_s)) super end - # uniq! keeps the first occurance of each path, otherwise preserving + # uniq! keeps the first occurrence of each path, otherwise preserving # order, preserving the effective load path def uniq!(*args) ret = super diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/core_ext/active_support.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/core_ext/active_support.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/core_ext/active_support.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/core_ext/active_support.rb 1970-01-01 00:00:00.000000000 +0000 @@ -1,107 +0,0 @@ -# frozen_string_literal: true -module Bootsnap - module LoadPathCache - module CoreExt - module ActiveSupport - def self.without_bootsnap_cache - prev = Thread.current[:without_bootsnap_cache] || false - Thread.current[:without_bootsnap_cache] = true - yield - ensure - Thread.current[:without_bootsnap_cache] = prev - end - - def self.allow_bootsnap_retry(allowed) - prev = Thread.current[:without_bootsnap_retry] || false - Thread.current[:without_bootsnap_retry] = !allowed - yield - ensure - Thread.current[:without_bootsnap_retry] = prev - end - - module ClassMethods - def autoload_paths=(o) - super - Bootsnap::LoadPathCache.autoload_paths_cache.reinitialize(o) - end - - def search_for_file(path) - return super if Thread.current[:without_bootsnap_cache] - begin - Bootsnap::LoadPathCache.autoload_paths_cache.find(path) - rescue Bootsnap::LoadPathCache::ReturnFalse - nil # doesn't really apply here - rescue Bootsnap::LoadPathCache::FallbackScan - nil # doesn't really apply here - end - end - - def autoloadable_module?(path_suffix) - Bootsnap::LoadPathCache.autoload_paths_cache.load_dir(path_suffix) - end - - def remove_constant(const) - CoreExt::ActiveSupport.without_bootsnap_cache { super } - end - - def require_or_load(*) - CoreExt::ActiveSupport.allow_bootsnap_retry(true) do - super - end - end - - # If we can't find a constant using the patched implementation of - # search_for_file, try again with the default implementation. - # - # These methods call search_for_file, and we want to modify its - # behaviour. The gymnastics here are a bit awkward, but it prevents - # 200+ lines of monkeypatches. - def load_missing_constant(from_mod, const_name) - CoreExt::ActiveSupport.allow_bootsnap_retry(false) do - super - end - rescue NameError => e - raise(e) if e.instance_variable_defined?(Bootsnap::LoadPathCache::ERROR_TAG_IVAR) - e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true) - - # This function can end up called recursively, we only want to - # retry at the top-level. - raise(e) if Thread.current[:without_bootsnap_retry] - # If we already had cache disabled, there's no use retrying - raise(e) if Thread.current[:without_bootsnap_cache] - # NoMethodError is a NameError, but we only want to handle actual - # NameError instances. - raise(e) unless e.class == NameError - # We can only confidently handle cases when *this* constant fails - # to load, not other constants referred to by it. - raise(e) unless e.name == const_name - # If the constant was actually loaded, something else went wrong? - raise(e) if from_mod.const_defined?(const_name) - CoreExt::ActiveSupport.without_bootsnap_cache { super } - end - - # Signature has changed a few times over the years; easiest to not - # reiterate it with version polymorphism here... - def depend_on(*) - super - rescue LoadError => e - raise(e) if e.instance_variable_defined?(Bootsnap::LoadPathCache::ERROR_TAG_IVAR) - e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true) - - # If we already had cache disabled, there's no use retrying - raise(e) if Thread.current[:without_bootsnap_cache] - CoreExt::ActiveSupport.without_bootsnap_cache { super } - end - end - end - end - end -end - -module ActiveSupport - module Dependencies - class << self - prepend(Bootsnap::LoadPathCache::CoreExt::ActiveSupport::ClassMethods) - end - end -end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb 2021-11-24 16:27:37.000000000 +0000 @@ -38,36 +38,29 @@ rescue Bootsnap::LoadPathCache::ReturnFalse false rescue Bootsnap::LoadPathCache::FallbackScan - require_with_bootsnap_lfi(path) + fallback = true + ensure + if fallback + require_with_bootsnap_lfi(path) + end end alias_method(:require_relative_without_bootsnap, :require_relative) def require_relative(path) + location = caller_locations(1..1).first realpath = Bootsnap::LoadPathCache.realpath_cache.call( - caller_locations(1..1).first.absolute_path, path + location.absolute_path || location.path, path ) require(realpath) end alias_method(:load_without_bootsnap, :load) def load(path, wrap = false) - if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)) - return load_without_bootsnap(resolved, wrap) - end - - # load also allows relative paths from pwd even when not in $: - if File.exist?(relative = File.expand_path(path)) - return load_without_bootsnap(relative, wrap) + if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path, try_extensions: false)) + load_without_bootsnap(resolved, wrap) + else + load_without_bootsnap(path, wrap) end - - raise(Bootsnap::LoadPathCache::CoreExt.make_load_error(path)) - rescue LoadError => e - e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true) - raise(e) - rescue Bootsnap::LoadPathCache::ReturnFalse - false - rescue Bootsnap::LoadPathCache::FallbackScan - load_without_bootsnap(path, wrap) end end @@ -88,6 +81,10 @@ rescue Bootsnap::LoadPathCache::ReturnFalse false rescue Bootsnap::LoadPathCache::FallbackScan - autoload_without_bootsnap(const, path) + fallback = true + ensure + if fallback + autoload_without_bootsnap(const, path) + end end end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/loaded_features_index.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/loaded_features_index.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/loaded_features_index.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/loaded_features_index.rb 2021-11-24 16:27:37.000000000 +0000 @@ -26,7 +26,7 @@ class LoadedFeaturesIndex def initialize @lfi = {} - @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped. + @mutex = Mutex.new # In theory the user could mutate $LOADED_FEATURES and invalidate our # cache. If this ever comes up in practice — or if you, the @@ -58,9 +58,9 @@ end def purge_multi(features) - rejected_hashes = features.map(&:hash).to_set + rejected_hashes = features.each_with_object({}) { |f, h| h[f.hash] = true } @mutex.synchronize do - @lfi.reject! { |_, hash| rejected_hashes.include?(hash) } + @lfi.reject! { |_, hash| rejected_hashes.key?(hash) } end end @@ -84,10 +84,18 @@ # entry. def register(short, long = nil) if long.nil? - pat = %r{/#{Regexp.escape(short)}(\.[^/]+)?$} len = $LOADED_FEATURES.size ret = yield - long = $LOADED_FEATURES[len..-1].detect { |feat| feat =~ pat } + long = $LOADED_FEATURES[len..-1].detect do |feat| + offset = 0 + while offset = feat.index(short, offset) + if feat.index(".", offset + 1) && !feat.index("/", offset + 2) + break true + else + offset += 1 + end + end + end else ret = yield end @@ -99,7 +107,7 @@ altname = if extension_elidable?(short) # Strip the extension off, e.g. 'bundler.rb' -> 'bundler'. strip_extension_if_elidable(short) - elsif long && (ext = File.extname(long)) + elsif long && (ext = File.extname(long.freeze)) # We already know the extension of the actual file this # resolves to, so put that back on. short + ext @@ -129,7 +137,7 @@ # to name files in a way that assumes otherwise. # (E.g. It's unlikely that someone will know that their code # will _never_ run on MacOS, and therefore think they can get away - # with callling a Ruby file 'x.dylib.rb' and then requiring it as 'x.dylib'.) + # with calling a Ruby file 'x.dylib.rb' and then requiring it as 'x.dylib'.) # # See . def extension_elidable?(f) diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/path.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/path.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/path.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/path.rb 2021-11-24 16:27:37.000000000 +0000 @@ -21,7 +21,7 @@ attr_reader(:path) def initialize(path) - @path = path.to_s + @path = path.to_s.freeze end # True if the path exists, but represents a non-directory object @@ -60,7 +60,7 @@ end def expanded_path - File.expand_path(path) + File.expand_path(path).freeze end private diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/path_scanner.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/path_scanner.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/path_scanner.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/path_scanner.rb 2021-11-24 16:27:37.000000000 +0000 @@ -5,7 +5,6 @@ module Bootsnap module LoadPathCache module PathScanner - ALL_FILES = "/{,**/*/**/}*" REQUIRABLE_EXTENSIONS = [DOT_RB] + DL_EXTENSIONS NORMALIZE_NATIVE_EXTENSIONS = !DL_EXTENSIONS.include?(LoadPathCache::DOT_SO) ALTERNATIVE_NATIVE_EXTENSIONS_PATTERN = /\.(o|bundle|dylib)\z/ @@ -16,34 +15,59 @@ '' end - def self.call(path) - path = path.to_s + class << self + def call(path) + path = File.expand_path(path.to_s).freeze + return [[], []] unless File.directory?(path) + + # If the bundle path is a descendent of this path, we do additional + # checks to prevent recursing into the bundle path as we recurse + # through this path. We don't want to scan the bundle path because + # anything useful in it will be present on other load path items. + # + # This can happen if, for example, the user adds '.' to the load path, + # and the bundle path is '.bundle'. + contains_bundle_path = BUNDLE_PATH.start_with?(path) + + dirs = [] + requirables = [] + walk(path, nil) do |relative_path, absolute_path, is_directory| + if is_directory + dirs << os_path(relative_path) + !contains_bundle_path || !absolute_path.start_with?(BUNDLE_PATH) + elsif relative_path.end_with?(*REQUIRABLE_EXTENSIONS) + requirables << os_path(relative_path) + end + end + [requirables, dirs] + end - relative_slice = (path.size + 1)..-1 - # If the bundle path is a descendent of this path, we do additional - # checks to prevent recursing into the bundle path as we recurse - # through this path. We don't want to scan the bundle path because - # anything useful in it will be present on other load path items. - # - # This can happen if, for example, the user adds '.' to the load path, - # and the bundle path is '.bundle'. - contains_bundle_path = BUNDLE_PATH.start_with?(path) - - dirs = [] - requirables = [] - - Dir.glob(path + ALL_FILES).each do |absolute_path| - next if contains_bundle_path && absolute_path.start_with?(BUNDLE_PATH) - relative_path = absolute_path.slice(relative_slice) - - if File.directory?(absolute_path) - dirs << relative_path - elsif REQUIRABLE_EXTENSIONS.include?(File.extname(relative_path)) - requirables << relative_path + def walk(absolute_dir_path, relative_dir_path, &block) + Dir.foreach(absolute_dir_path) do |name| + next if name.start_with?('.') + relative_path = relative_dir_path ? File.join(relative_dir_path, name) : name + + absolute_path = "#{absolute_dir_path}/#{name}" + if File.directory?(absolute_path) + if yield relative_path, absolute_path, true + walk(absolute_path, relative_path, &block) + end + else + yield relative_path, absolute_path, false + end end end - [requirables, dirs] + if RUBY_VERSION >= '3.1' + def os_path(path) + path.freeze + end + else + def os_path(path) + path.force_encoding(Encoding::US_ASCII) if path.ascii_only? + path.freeze + end + end end end end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/realpath_cache.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/realpath_cache.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/realpath_cache.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/realpath_cache.rb 2021-11-24 16:27:37.000000000 +0000 @@ -15,15 +15,15 @@ def realpath(caller_location, path) base = File.dirname(caller_location) - file = find_file(File.expand_path(path, base)) - dir = File.dirname(file) - File.join(dir, File.basename(file)) + abspath = File.expand_path(path, base).freeze + find_file(abspath) end def find_file(name) - ['', *CACHED_EXTENSIONS].each do |ext| + return File.realpath(name).freeze if File.exist?(name) + CACHED_EXTENSIONS.each do |ext| filename = "#{name}#{ext}" - return File.realpath(filename) if File.exist?(filename) + return File.realpath(filename).freeze if File.exist?(filename) end name end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/store.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/store.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache/store.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache/store.rb 2021-11-24 16:27:37.000000000 +0000 @@ -12,8 +12,7 @@ def initialize(store_path) @store_path = store_path - # TODO: Remove conditional once Ruby 2.2 support is dropped. - @txn_mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new + @txn_mutex = Mutex.new @dirty = false load_data end @@ -63,12 +62,18 @@ def load_data @data = begin - MessagePack.load(File.binread(@store_path)) - # handle malformed data due to upgrade incompatability - rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError - {} - rescue ArgumentError => e - e.message =~ /negative array size/ ? {} : raise + File.open(@store_path, encoding: Encoding::BINARY) do |io| + MessagePack.load(io) + end + # handle malformed data due to upgrade incompatibility + rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError + {} + rescue ArgumentError => error + if error.message =~ /negative array size/ + {} + else + raise + end end end @@ -80,10 +85,13 @@ exclusive_write = File::Constants::CREAT | File::Constants::EXCL | File::Constants::WRONLY # `encoding:` looks redundant wrt `binwrite`, but necessary on windows # because binary is part of mode. - File.binwrite(tmp, MessagePack.dump(@data), mode: exclusive_write, encoding: Encoding::BINARY) + File.open(tmp, mode: exclusive_write, encoding: Encoding::BINARY) do |io| + MessagePack.dump(@data, io, freeze: true) + end FileUtils.mv(tmp, @store_path) rescue Errno::EEXIST retry + rescue SystemCallError end end end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache.rb ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/load_path_cache.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/load_path_cache.rb 2021-11-24 16:27:37.000000000 +0000 @@ -28,10 +28,9 @@ CACHED_EXTENSIONS = DLEXT2 ? [DOT_RB, DLEXT, DLEXT2] : [DOT_RB, DLEXT] class << self - attr_reader(:load_path_cache, :autoload_paths_cache, - :loaded_features_index, :realpath_cache) + attr_reader(:load_path_cache, :loaded_features_index, :realpath_cache) - def setup(cache_path:, development_mode:, active_support: true) + def setup(cache_path:, development_mode:) unless supported? warn("[bootsnap/setup] Load path caching is not supported on this implementation of Ruby") if $VERBOSE return @@ -45,23 +44,11 @@ @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode) require_relative('load_path_cache/core_ext/kernel_require') require_relative('load_path_cache/core_ext/loaded_features') - - if active_support - # this should happen after setting up the initial cache because it - # loads a lot of code. It's better to do after +require+ is optimized. - require('active_support/dependencies') - @autoload_paths_cache = Cache.new( - store, - ::ActiveSupport::Dependencies.autoload_paths, - development_mode: development_mode - ) - require_relative('load_path_cache/core_ext/active_support') - end end def supported? RUBY_ENGINE == 'ruby' && - RUBY_PLATFORM =~ /darwin|linux|bsd/ + RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/ end end end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/setup.rb ruby-bootsnap-1.9.3/lib/bootsnap/setup.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/setup.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/setup.rb 2021-11-24 16:27:37.000000000 +0000 @@ -1,39 +1,4 @@ # frozen_string_literal: true require_relative('../bootsnap') -env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV'] -development_mode = ['', nil, 'development'].include?(env) - -cache_dir = ENV['BOOTSNAP_CACHE_DIR'] -unless cache_dir - config_dir_frame = caller.detect do |line| - line.include?('/config/') - end - - unless config_dir_frame - $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:") - $stderr.puts("[bootsnap/setup] 1. require bootsnap/setup from your application's config directory; or") - $stderr.puts("[bootsnap/setup] 2. Define the environment variable BOOTSNAP_CACHE_DIR") - - raise("couldn't infer bootsnap cache directory") - end - - path = config_dir_frame.split(/:\d+:/).first - path = File.dirname(path) until File.basename(path) == 'config' - app_root = File.dirname(path) - - cache_dir = File.join(app_root, 'tmp', 'cache') -end - -ruby_version = Gem::Version.new(RUBY_VERSION) -iseq_cache_enabled = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0') - -Bootsnap.setup( - cache_dir: cache_dir, - development_mode: development_mode, - load_path_cache: true, - autoload_paths_cache: true, # assume rails. open to PRs to impl. detection - disable_trace: false, - compile_cache_iseq: iseq_cache_enabled, - compile_cache_yaml: true, -) +Bootsnap.default_setup diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap/version.rb ruby-bootsnap-1.9.3/lib/bootsnap/version.rb --- ruby-bootsnap-1.4.6/lib/bootsnap/version.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap/version.rb 2021-11-24 16:27:37.000000000 +0000 @@ -1,4 +1,4 @@ # frozen_string_literal: true module Bootsnap - VERSION = "1.4.6" + VERSION = "1.9.3" end diff -Nru ruby-bootsnap-1.4.6/lib/bootsnap.rb ruby-bootsnap-1.9.3/lib/bootsnap.rb --- ruby-bootsnap-1.4.6/lib/bootsnap.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/lib/bootsnap.rb 2021-11-24 16:27:37.000000000 +0000 @@ -1,4 +1,5 @@ # frozen_string_literal: true + require_relative('bootsnap/version') require_relative('bootsnap/bundler') require_relative('bootsnap/load_path_cache') @@ -7,42 +8,119 @@ module Bootsnap InvalidConfiguration = Class.new(StandardError) + class << self + attr_reader :logger + end + + def self.log! + self.logger = $stderr.method(:puts) + end + + def self.logger=(logger) + @logger = logger + if logger.respond_to?(:debug) + self.instrumentation = ->(event, path) { @logger.debug("[Bootsnap] #{event} #{path}") } + else + self.instrumentation = ->(event, path) { @logger.call("[Bootsnap] #{event} #{path}") } + end + end + + def self.instrumentation=(callback) + @instrumentation = callback + if respond_to?(:instrumentation_enabled=, true) + self.instrumentation_enabled = !!callback + end + end + + def self._instrument(event, path) + @instrumentation.call(event, path) + end + def self.setup( cache_dir:, development_mode: true, load_path_cache: true, - autoload_paths_cache: true, - disable_trace: false, + autoload_paths_cache: nil, + disable_trace: nil, compile_cache_iseq: true, - compile_cache_yaml: true + compile_cache_yaml: true, + compile_cache_json: true ) - if autoload_paths_cache && !load_path_cache - raise(InvalidConfiguration, "feature 'autoload_paths_cache' depends on feature 'load_path_cache'") + unless autoload_paths_cache.nil? + warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \ + "If you use Zeitwerk this option is useless, and if you are still using the classic autoloader " \ + "upgrading is recommended." end - setup_disable_trace if disable_trace + unless disable_trace.nil? + warn "[DEPRECATED] Bootsnap's `disable_trace:` option is deprecated and will be removed. " \ + "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended." + end + + if compile_cache_iseq && !iseq_cache_supported? + warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommened " \ + "to turn `compile_cache_iseq` off on Ruby 2.5" + end Bootsnap::LoadPathCache.setup( - cache_path: cache_dir + '/bootsnap-load-path-cache', + cache_path: cache_dir + '/bootsnap/load-path-cache', development_mode: development_mode, - active_support: autoload_paths_cache ) if load_path_cache Bootsnap::CompileCache.setup( - cache_dir: cache_dir + '/bootsnap-compile-cache', + cache_dir: cache_dir + '/bootsnap/compile-cache', iseq: compile_cache_iseq, - yaml: compile_cache_yaml + yaml: compile_cache_yaml, + json: compile_cache_json, ) end - def self.setup_disable_trace - if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.5.0') - warn( - "from #{caller_locations(1, 1)[0]}: The 'disable_trace' method is not allowed with this Ruby version. " \ - "current: #{RUBY_VERSION}, allowed version: < 2.5.0", + def self.iseq_cache_supported? + return @iseq_cache_supported if defined? @iseq_cache_supported + + ruby_version = Gem::Version.new(RUBY_VERSION) + @iseq_cache_supported = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0') + end + + def self.default_setup + env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV'] + development_mode = ['', nil, 'development'].include?(env) + + unless ENV['DISABLE_BOOTSNAP'] + cache_dir = ENV['BOOTSNAP_CACHE_DIR'] + unless cache_dir + config_dir_frame = caller.detect do |line| + line.include?('/config/') + end + + unless config_dir_frame + $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:") + $stderr.puts("[bootsnap/setup] 1. require bootsnap/setup from your application's config directory; or") + $stderr.puts("[bootsnap/setup] 2. Define the environment variable BOOTSNAP_CACHE_DIR") + + raise("couldn't infer bootsnap cache directory") + end + + path = config_dir_frame.split(/:\d+:/).first + path = File.dirname(path) until File.basename(path) == 'config' + app_root = File.dirname(path) + + cache_dir = File.join(app_root, 'tmp', 'cache') + end + + + setup( + cache_dir: cache_dir, + development_mode: development_mode, + load_path_cache: !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'], + compile_cache_iseq: !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_supported?, + compile_cache_yaml: !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'], + compile_cache_json: !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'], ) - else - RubyVM::InstructionSequence.compile_option = { trace_instruction: false } + + if ENV['BOOTSNAP_LOG'] + log! + end end end end diff -Nru ruby-bootsnap-1.4.6/Rakefile ruby-bootsnap-1.9.3/Rakefile --- ruby-bootsnap-1.4.6/Rakefile 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/Rakefile 2021-11-24 16:27:37.000000000 +0000 @@ -10,4 +10,12 @@ ext.gem_spec = gemspec end -task(default: :compile) +require "rake/testtask" + +Rake::TestTask.new(:test) do |t| + t.libs << "test" + t.libs << "lib" + t.test_files = FileList["test/**/*_test.rb"] +end + +task(default: %i(compile test)) diff -Nru ruby-bootsnap-1.4.6/README.jp.md ruby-bootsnap-1.9.3/README.jp.md --- ruby-bootsnap-1.4.6/README.jp.md 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/README.jp.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,231 +0,0 @@ -# Bootsnap [![Build Status](https://travis-ci.org/Shopify/bootsnap.svg?branch=master)](https://travis-ci.org/Shopify/bootsnap) - -Bootsnap は RubyVM におけるバイトコード生成やファイルルックアップ等の時間のかかる処理を最適化するためのライブラリです。ActiveSupport や YAML もサポートしています。[内部動作](#内部動作)もご覧ください。 - -注意書き: このライブラリは英語話者によって管理されています。この README は日本語ですが、日本語でのサポートはしておらず、リクエストにお答えすることもできません。バイリンガルの方がサポートをサポートしてくださる場合はお知らせください！:) - -### パフォーマンス - -* [Discourse](https://github.com/discourse/discourse) では、約6秒から3秒まで、約50%の起動時間短縮が確認されています。 -* 小さなアプリケーションでも、50%の改善（3.6秒から1.8秒）が確認されています。 -* 非常に巨大でモノリシックなアプリである Shopify のプラットフォームでは、約25秒から6.5秒へと約75％短縮されました。 - -## 使用方法 - -この gem は macOS と Linux で作動します。まずは、`bootsnap` を `Gemfile` に追加します: - -```ruby -gem 'bootsnap', require: false -``` - -Rails を使用している場合は、以下のコードを、`config/boot.rb` 内にある `require 'bundler/setup'` の直後に追加してください。 - -```ruby -require 'bootsnap/setup' -``` - -単に `gem 'bootsnap', require: 'bootsnap/setup'` と指定することも技術的には可能ですが、最大限のパフォーマンス改善を得るためには Bootsnap をできるだけ早く読み込むことが重要です。 - -この require の仕組みは[こちら](https://github.com/Shopify/bootsnap/blob/master/lib/bootsnap/setup.rb)で確認できます。 - -Rails を使用していない場合、または、より多くの設定を変更したい場合は、以下のコードを `require 'bundler/setup'` の直後に追加してください(早く読み込まれるほど、より多くのものを最適化することができます）。 - -```ruby -require 'bootsnap' -env = ENV['RAILS_ENV'] || "development" -Bootsnap.setup( -  cache_dir:           'tmp/cache',         # キャッシュファイルを保存する path -  development_mode:     env == 'development', # 現在の作業環境、例えば RACK_ENV, RAILS_ENV など。 - load_path_cache: true, # キャッシュで LOAD_PATH を最適化する。 -  autoload_paths_cache: true,                 # キャッシュで ActiveSupport による autoload を行う。 -  disable_trace:       true,                 # (アルファ) `RubyVM::InstructionSequence.compile_option = { trace_instruction: false }`をセットする。 -  compile_cache_iseq:   true,                 # ISeq キャッシュをコンパイルする -  compile_cache_yaml:   true                 # YAML キャッシュをコンパイルする -) -``` - -**ヒント**: `require 'bootsnap'` を `BootLib::Require.from_gem('bootsnap', 'bootsnap')` で、 [こちらのトリック](https://github.com/Shopify/bootsnap/wiki/Bootlib::Require)を使って置き換えることができます。こうすると、巨大な`$LOAD_PATH`がある場合でも、起動時間を最短化するのに役立ちます。 - -注意: Bootsnap と [Spring](https://github.com/rails/spring) は別領域の問題を扱うツールです。Bootsnap は個々のソースファイルの読み込みを高速化します。一方で、Spring は起動されたRailsプロセスのコピーを保持して次回の起動時に起動プロセスの一部を完全にスキップします。2つのツールはうまく連携しており、どちらも新しく生成された Rails アプリケーションにデフォルトで含まれています。 - -### 環境 -Bootsnapのすべての機能はセットアップ時の設定に従って開発、テスト、プロダクション、および他のすべての環境で有効化されます。Shopify では、この gem を問題なくすべての環境で安全に使用しています。 - -特定の環境で機能を無効にする場合は、必要に応じて適切な ENV 変数または設定を考慮して設定を変更することをおすすめします。 - -## 内部動作 - -Bootsnap は、処理に時間のかかるメソッドの結果をキャッシュすることで最適化しています。これは、大きく分けて2つのカテゴリに分けられます。 - -* [Path Pre-Scanning](#path-pre-scanning) - * `Kernel#require` と `Kernel#load` を `$LOAD_PATH` フルスキャンを行わないように変更します。 - * `ActiveSupport::Dependencies.{autoloadable_module?,load_missing_constant,depend_on}` を `ActiveSupport::Dependencies.autoload_paths` のフルスキャンを行わないようにオーバーライドします。 -* [Compilation caching](#compilation-caching) - * Ruby バイトコードのコンパイル結果をキャッシュするためのメソッド `RubyVM::InstructionSequence.load_iseq` が実装されています。 - * `YAML.load_file` を YAML オブジェクトのロード結果を MessagePack でキャッシュするように変更します。 MessagePack でサポートされていないタイプが使われている場合は Marshal が使われます。 - -### Path Pre-Scanning - -_(このライブラリは [bootscale](https://github.com/byroot/bootscale) という別のライブラリを元に開発されました)_ - -Bootsnap の初期化時、あるいはパス(例えば、`$LOAD_PATH`)の変更時に、`Bootsnap::LoadPathCache` がキャッシュから必要なエントリーのリストを読み込みます。または、必要に応じてフルスキャンを実行し結果をキャッシュします。 -その後、たとえば `require 'foo'` を評価する場合, Ruby は `$LOAD_PATH` `['x', 'y', ...]` のすべてのエントリーを繰り返し評価することで `x/foo.rb`, `y/foo.rb` などを探索します。これに対して Bootsnap は、キャッシュされた require 可能なファイルと `$LOAD_PATH` を見ることで、Rubyが最終的に選択するであろうパスで置き換えます。 - -この動作によって生成された syscall を見ると、最終的な結果は以前なら次のようになります。 - -``` -open x/foo.rb # (fail) -# (imagine this with 500 $LOAD_PATH entries instead of two) -open y/foo.rb # (success) -close y/foo.rb -open y/foo.rb -... -``` - -これが、次のようになります: - -``` -open y/foo.rb -... -``` - -`autoload_paths_cache` オプションが `Bootsnap.setup` に与えられている場合、`ActiveSupport::Dependencies.autoload_paths` をトラバースする方法にはまったく同じ最適化が使用されます。 - -`*_path_cache` を機能させるオーバーライドを図にすると、次のようになります。 - -![Bootsnapの説明図](https://cloud.githubusercontent.com/assets/3074765/24532120/eed94e64-158b-11e7-9137-438d759b2ac8.png) - -Bootsnap は、 `$LOAD_PATH` エントリを安定エントリと不安定エントリの2つのカテゴリに分類します。不安定エントリはアプリケーションが起動するたびにスキャンされ、そのキャッシュは30秒間だけ有効になります。安定エントリーに期限切れはありません。コンテンツがスキャンされると、決して変更されないものとみなされます。 - -安定していると考えられる唯一のディレクトリは、Rubyのインストールプレフィックス (`RbConfig::CONFIG['prefix']`, または `/usr/local/ruby` や `~/.rubies/x.y.z`)下にあるものと、`Gem.path` (たとえば　`~/.gem/ruby/x.y.z`) や `Bundler.bundle_path` 下にあるものです。他のすべては不安定エントリと分類されます。 - -[`Bootsnap::LoadPathCache::Cache`](https://github.com/Shopify/bootsnap/blob/master/lib/bootsnap/load_path_cache/cache.rb) に加えて次の図では、エントリの解決がどのように機能するかを理解するのに役立つかもしれません。経路探索は以下のようになります。 - -![パス探索の仕組み](https://cloud.githubusercontent.com/assets/3074765/25388270/670b5652-299b-11e7-87fb-975647f68981.png) - -また、`LoadError` のスキャンがどれほど重いかに注意を払うことも大切です。もし Ruby が `require 'something'` を評価し、そのファイルが `$LOAD_PATH` にない場合は、それを知るために `2 * $LOAD_PATH.length` のファイルシステムアスセスが必要になります。Bootsnap は、ファイルシステムにまったく触れずに `LoadError` を投げ、この結果をキャッシュします。 - -## Compilation Caching - -*（このコンセプトのより分かりやすい解説は [yomikomu](https://github.com/ko1/yomikomu) をお読み下さい。)* - -Ruby には複雑な文法が実装されており、構文解析は簡単なオペレーションではありません。1.9以降、Ruby は Ruby ソースを内部のバイトコードに変換した後、Ruby VM によって実行してきました。2.3.0 以降、[RubyはAPIを公開し](https://ruby-doc.org/core-2.3.0/RubyVM/InstructionSequence.html)、そのバイトコードをキャッシュすることができるようになりました。これにより、同じファイルが複数ロードされた時の、比較的時間のかかる部分をバイパスすることができます。 - -また、アプリケーションの起動時に YAML ドキュメントの読み込みに多くの時間を費やしていることを発見しました。そして、 MessagePack と Marshal は deserialization にあたって YAML よりもはるかに高速であるということに気付きました。そこで、YAML ドキュメントを、Ruby バイトコードと同じコンパイルキャッシングの最適化を施すことで、高速化しています。Ruby の "バイトコード" フォーマットに相当するものは MessagePack ドキュメント (あるいは、MessagePack をサポートしていないタイプの YAML ドキュメントの場合は、Marshal stream)になります。 - -これらのコンパイル結果は、入力ファイル（FNV1a-64）のフルパスのハッシュを取って生成されたファイル名で、キャッシュディレクトリに保存されます。 - -Bootsnap 無しでは、ファイルを `require` するために生成された syscall の順序は次のようになっていました: - -``` -open /c/foo.rb -> m -fstat64 m -close m -open /c/foo.rb -> o -fstat64 o -fstat64 o -read o -read o -... -close o -``` - -しかし Bootsnap では、次のようになります: - -``` -open /c/foo.rb -> n -fstat64 n -close n -open /c/foo.rb -> n -fstat64 n -open (cache) -> m -read m -read m -close m -close n -``` - -これは一見劣化していると思われるかもしれませんが、性能に大きな違いがあります。 - -*（両方のリストの最初の3つの syscalls -- `open`, `fstat64`, `close` -- は本質的に有用ではありません。[このRubyパッチ](https://bugs.ruby-lang.org/issues/13378)は、Boosnap と組み合わせることによって、それらを最適化しています）* - -Bootsnap は、64バイトのヘッダーとそれに続くキャッシュの内容を含んだキャッシュファイルを書き込みます。ヘッダーは、次のいくつかのフィールドで構成されるキャッシュキーです。 - -- `version`、Bootsnapにハードコードされる基本的なスキーマのバージョン -- `ruby_platform`、`RUBY_PLATFORM`（x86_64-linux-gnuなど）変数とglibcバージョン（Linuxの場合）またはOSバージョン（BSD、macOSの場合は` uname -v`）のハッシュ -- `compile_option`、`RubyVM::InstructionSequence.compile_option` の返り値 -- `ruby_revision`、コンパイルされたRubyのバージョン -- `size`、ソースファイルのサイズ -- `mtime`、コンパイル時のソースファイルの最終変更タイムスタンプ -- `data_size`、バッファに読み込む必要のあるヘッダーに続くバイト数。 - -キーが有効な場合、キャッシュがファイルからロードされます。そうでない場合、キャッシュは再生成され、現在のキャッシュを破棄します。 - -# 最終的なキャッシュ結果 - -次のファイル構造があるとします。 - -``` -/ -├── a -├── b -└── c - └── foo.rb -``` - -そして、このような `$LOAD_PATH` があるとします。 - -``` -["/a", "/b", "/c"] -``` - -Bootsnap なしで `require 'foo'` を呼び出すと、Ruby は次の順序で syscalls を生成します: - -``` -open /a/foo.rb -> -1 -open /b/foo.rb -> -1 -open /c/foo.rb -> n -close n -open /c/foo.rb -> m -fstat64 m -close m -open /c/foo.rb -> o -fstat64 o -fstat64 o -read o -read o -... -close o -``` - -しかし Bootsnap では、次のようになります: - -``` -open /c/foo.rb -> n -fstat64 n -close n -open /c/foo.rb -> n -fstat64 n -open (cache) -> m -read m -read m -close m -close n -``` - -Bootsnap なしで `require 'nope'` を呼び出すと、次のようになります: - -``` -open /a/nope.rb -> -1 -open /b/nope.rb -> -1 -open /c/nope.rb -> -1 -open /a/nope.bundle -> -1 -open /b/nope.bundle -> -1 -open /c/nope.bundle -> -1 -``` - -...そして、Bootsnap で `require 'nope'` を呼び出すと、次のようになります... - -``` -# (nothing!) -``` diff -Nru ruby-bootsnap-1.4.6/README.md ruby-bootsnap-1.9.3/README.md --- ruby-bootsnap-1.4.6/README.md 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/README.md 2021-11-24 16:27:37.000000000 +0000 @@ -1,6 +1,6 @@ -# Bootsnap [![Build Status](https://travis-ci.org/Shopify/bootsnap.svg?branch=master)](https://travis-ci.org/Shopify/bootsnap) +# Bootsnap [![Actions Status](https://github.com/Shopify/bootsnap/workflows/ci/badge.svg)](https://github.com/Shopify/bootsnap/actions) -Bootsnap is a library that plugs into Ruby, with optional support for `ActiveSupport` and `YAML`, +Bootsnap is a library that plugs into Ruby, with optional support for `YAML`, to optimize and cache expensive computations. See [How Does This Work](#how-does-this-work). #### Performance @@ -11,7 +11,7 @@ - The core Shopify platform -- a rather large monolithic application -- boots about 75% faster, dropping from around 25s to 6.5s. * In Shopify core (a large app), about 25% of this gain can be attributed to `compile_cache_*` - features; 75% to path caching, and ~1% to `disable_trace`. This is fairly representative. + features; 75% to path caching. This is fairly representative. ## Usage @@ -29,7 +29,8 @@ require 'bootsnap/setup' ``` -Note that bootsnap writes to `tmp/cache`, and that directory *must* be writable. Rails will fail to +Note that bootsnap writes to `tmp/cache` (or the path specified by `ENV['BOOTSNAP_CACHE_DIR']`), +and that directory *must* be writable. Rails will fail to boot if it is not. If this is unacceptable (e.g. you are running in a read-only container and unwilling to mount in a writable tmpdir), you should remove this line or wrap it in a conditional. @@ -53,15 +54,11 @@ cache_dir: 'tmp/cache', # Path to your cache development_mode: env == 'development', # Current working environment, e.g. RACK_ENV, RAILS_ENV, etc load_path_cache: true, # Optimize the LOAD_PATH with a cache - autoload_paths_cache: true, # Optimize ActiveSupport autoloads with cache - disable_trace: true, # Set `RubyVM::InstructionSequence.compile_option = { trace_instruction: false }` compile_cache_iseq: true, # Compile Ruby code into ISeq cache, breaks coverage reporting. compile_cache_yaml: true # Compile YAML into a cache ) ``` -**Note that `disable_trace` will break debuggers and tracing.** - **Protip:** You can replace `require 'bootsnap'` with `BootLib::Require.from_gem('bootsnap', 'bootsnap')` using [this trick](https://github.com/Shopify/bootsnap/wiki/Bootlib::Require). This will help optimize boot time further if you have an extremely large `$LOAD_PATH`. @@ -71,12 +68,39 @@ on hand to completely skip parts of the boot process the next time it's needed. The two tools work well together, and are both included in a newly-generated Rails applications by default. +### Environment variables + +`require 'bootsnap/setup'` behavior can be changed using environment variables: + +- `BOOTSNAP_CACHE_DIR` allows to define the cache location. +- `DISABLE_BOOTSNAP` allows to entirely disable bootsnap. +- `DISABLE_BOOTSNAP_LOAD_PATH_CACHE` allows to disable load path caching. +- `DISABLE_BOOTSNAP_COMPILE_CACHE` allows to disable ISeq and YAML caches. +- `BOOTSNAP_LOG` configure bootsnap to log all caches misses to STDERR. + ### Environments All Bootsnap features are enabled in development, test, production, and all other environments according to the configuration in the setup. At Shopify, we use this gem safely in all environments without issue. If you would like to disable any feature for a certain environment, we suggest changing the configuration to take into account the appropriate ENV var or configuration according to your needs. +### Instrumentation + +Bootsnap cache misses can be monitored though a callback: + +```ruby +Bootsnap.instrumentation = ->(event, path) { puts "#{event} #{path}" } +``` + +`event` is either `:miss` or `:stale`. You can also call `Bootsnap.log!` as a shortcut to +log all events to STDERR. + +To turn instrumentation back off you can set it to nil: + +```ruby +Bootsnap.instrumentation = nil +``` + ## How does this work? Bootsnap optimizes methods to cache results of expensive computations, and can be grouped @@ -84,8 +108,6 @@ * [Path Pre-Scanning](#path-pre-scanning) * `Kernel#require` and `Kernel#load` are modified to eliminate `$LOAD_PATH` scans. - * `ActiveSupport::Dependencies.{autoloadable_module?,load_missing_constant,depend_on}` are - overridden to eliminate scans of `ActiveSupport::Dependencies.autoload_paths`. * [Compilation caching](#compilation-caching) * `RubyVM::InstructionSequence.load_iseq` is implemented to cache the result of ruby bytecode compilation. @@ -124,10 +146,6 @@ ... ``` -Exactly the same strategy is employed for methods that traverse -`ActiveSupport::Dependencies.autoload_paths` if the `autoload_paths_cache` option is given to -`Bootsnap.setup`. - The following diagram flowcharts the overrides that make the `*_path_cache` features work. ![Flowchart explaining @@ -294,6 +312,19 @@ # (nothing!) ``` +## Precompilation + +In development environments the bootsnap compilation cache is generated on the fly when source files are loaded. +But in production environments, such as docker images, you might need to precompile the cache. + +To do so you can use the `bootsnap precompile` command. + +Example: + +```bash +$ bundle exec bootsnap precompile --gemfile app/ lib/ +``` + ## When not to use Bootsnap *Alternative engines*: Bootsnap is pretty reliant on MRI features, and parts are disabled entirely on alternative ruby diff -Nru ruby-bootsnap-1.4.6/.rubocop.yml ruby-bootsnap-1.9.3/.rubocop.yml --- ruby-bootsnap-1.4.6/.rubocop.yml 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/.rubocop.yml 2021-11-24 16:27:37.000000000 +0000 @@ -5,10 +5,10 @@ Exclude: - 'vendor/**/*' - 'tmp/**/*' - TargetRubyVersion: '2.3' + TargetRubyVersion: '2.4' # This doesn't take into account retrying from an exception -Lint/HandleExceptions: +Lint/SuppressedException: Enabled: false # allow String.new to create mutable strings diff -Nru ruby-bootsnap-1.4.6/test/cli_test.rb ruby-bootsnap-1.9.3/test/cli_test.rb --- ruby-bootsnap-1.4.6/test/cli_test.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/cli_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,59 @@ +# frozen_string_literal: true +require('test_helper') +require('bootsnap/cli') + +module Bootsnap + class CLITest < Minitest::Test + include(TmpdirHelper) + + def setup + super + @cache_dir = File.expand_path('tmp/cache/bootsnap/compile-cache') + end + + def test_precompile_single_file + path = Help.set_file('a.rb', 'a = a = 3', 100) + CompileCache::ISeq.expects(:precompile).with(File.expand_path(path), cache_dir: @cache_dir) + assert_equal 0, CLI.new(['precompile', '-j', '0', path]).run + end + + def test_no_iseq + path = Help.set_file('a.rb', 'a = a = 3', 100) + CompileCache::ISeq.expects(:precompile).never + assert_equal 0, CLI.new(['precompile', '-j', '0', '--no-iseq', path]).run + end + + def test_precompile_directory + path_a = Help.set_file('foo/a.rb', 'a = a = 3', 100) + path_b = Help.set_file('foo/b.rb', 'b = b = 3', 100) + + CompileCache::ISeq.expects(:precompile).with(File.expand_path(path_a), cache_dir: @cache_dir) + CompileCache::ISeq.expects(:precompile).with(File.expand_path(path_b), cache_dir: @cache_dir) + assert_equal 0, CLI.new(['precompile', '-j', '0', 'foo']).run + end + + def test_precompile_exclude + path_a = Help.set_file('foo/a.rb', 'a = a = 3', 100) + Help.set_file('foo/b.rb', 'b = b = 3', 100) + + CompileCache::ISeq.expects(:precompile).with(File.expand_path(path_a), cache_dir: @cache_dir) + assert_equal 0, CLI.new(['precompile', '-j', '0', '--exclude', 'b.rb', 'foo']).run + end + + def test_precompile_gemfile + assert_equal 0, CLI.new(['precompile', '--gemfile']).run + end + + def test_precompile_yaml + path = Help.set_file('a.yaml', 'foo: bar', 100) + CompileCache::YAML.expects(:precompile).with(File.expand_path(path), cache_dir: @cache_dir) + assert_equal 0, CLI.new(['precompile', '-j', '0', path]).run + end + + def test_no_yaml + path = Help.set_file('a.yaml', 'foo: bar', 100) + CompileCache::YAML.expects(:precompile).never + assert_equal 0, CLI.new(['precompile', '-j', '0', '--no-yaml', path]).run + end + end +end diff -Nru ruby-bootsnap-1.4.6/test/compile_cache/iseq_cache_test.rb ruby-bootsnap-1.9.3/test/compile_cache/iseq_cache_test.rb --- ruby-bootsnap-1.4.6/test/compile_cache/iseq_cache_test.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/compile_cache/iseq_cache_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,11 @@ +# frozen_string_literal: true +require('test_helper') + +class CompileCacheISeqTest < Minitest::Test + include(TmpdirHelper) + + def test_ruby_bug_18250 + Help.set_file('a.rb', 'def foo(*); ->{ super }; end; def foo(**); ->{ super }; end', 100) + Bootsnap::CompileCache::ISeq.fetch('a.rb') + end +end diff -Nru ruby-bootsnap-1.4.6/test/compile_cache/json_test.rb ruby-bootsnap-1.9.3/test/compile_cache/json_test.rb --- ruby-bootsnap-1.4.6/test/compile_cache/json_test.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/compile_cache/json_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,81 @@ +# frozen_string_literal: true +require('test_helper') + +class CompileCacheJSONTest < Minitest::Test + include(TmpdirHelper) + + module FakeJson + Fallback = Class.new(StandardError) + class << self + def load_file(path, symbolize_names: false, freeze: false, fallback: nil) + raise Fallback + end + end + end + + def setup + super + Bootsnap::CompileCache::JSON.init! + FakeJson.singleton_class.prepend(Bootsnap::CompileCache::JSON::Patch) + end + + def test_json_input_to_output + document = ::Bootsnap::CompileCache::JSON.input_to_output(<<~JSON, {}) + { + "foo": 42, + "bar": [1] + } + JSON + expected = { + 'foo' => 42, + 'bar' => [1], + } + assert_equal expected, document + end + + def test_load_file + Help.set_file('a.json', '{"foo": "bar"}', 100) + assert_equal({'foo' => 'bar'}, FakeJson.load_file('a.json')) + end + + def test_load_file_symbolize_names + Help.set_file('a.json', '{"foo": "bar"}', 100) + FakeJson.load_file('a.json') + + if ::Bootsnap::CompileCache::JSON.supported_options.include?(:symbolize_names) + 2.times do + assert_equal({foo: 'bar'}, FakeJson.load_file('a.json', symbolize_names: true)) + end + else + assert_raises(FakeJson::Fallback) do # would call super + FakeJson.load_file('a.json', symbolize_names: true) + end + end + end + + def test_load_file_freeze + Help.set_file('a.json', '["foo"]', 100) + FakeJson.load_file('a.json') + + if ::Bootsnap::CompileCache::JSON.supported_options.include?(:freeze) + 2.times do + string = FakeJson.load_file('a.json', freeze: true).first + assert_equal("foo", string) + assert_predicate(string, :frozen?) + end + else + assert_raises(FakeJson::Fallback) do # would call super + FakeJson.load_file('a.json', freeze: true) + end + end + end + + def test_load_file_unknown_option + Help.set_file('a.json', '["foo"]', 100) + FakeJson.load_file('a.json') + + assert_raises(FakeJson::Fallback) do # would call super + FakeJson.load_file('a.json', fallback: true) + end + end +end diff -Nru ruby-bootsnap-1.4.6/test/compile_cache/yaml_test.rb ruby-bootsnap-1.9.3/test/compile_cache/yaml_test.rb --- ruby-bootsnap-1.4.6/test/compile_cache/yaml_test.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/compile_cache/yaml_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,132 @@ +# frozen_string_literal: true +require('test_helper') + +class CompileCacheYAMLTest < Minitest::Test + include(TmpdirHelper) + + module FakeYaml + Fallback = Class.new(StandardError) + class << self + def load_file(path, symbolize_names: false, freeze: false, fallback: nil) + raise Fallback + end + + def unsafe_load_file(path, symbolize_names: false, freeze: false, fallback: nil) + raise Fallback + end + end + end + + def setup + super + Bootsnap::CompileCache::YAML.init! + FakeYaml.singleton_class.prepend(Bootsnap::CompileCache::YAML::Patch) + end + + def test_yaml_strict_load + document = ::Bootsnap::CompileCache::YAML.strict_load(<<~YAML) + --- + :foo: 42 + bar: [1] + YAML + expected = { + foo: 42, + 'bar' => [1], + } + assert_equal expected, document + end + + def test_yaml_input_to_output + document = ::Bootsnap::CompileCache::YAML.input_to_output(<<~YAML, {}) + --- + :foo: 42 + bar: [1] + YAML + expected = { + foo: 42, + 'bar' => [1], + } + assert_equal expected, document + end + + def test_yaml_tags + error = assert_raises Bootsnap::CompileCache::Uncompilable do + ::Bootsnap::CompileCache::YAML.strict_load('!many Boolean') + end + assert_equal "YAML tags are not supported: !many", error.message + + error = assert_raises Bootsnap::CompileCache::Uncompilable do + ::Bootsnap::CompileCache::YAML.strict_load('!ruby/object {}') + end + assert_equal "YAML tags are not supported: !ruby/object", error.message + end + + if YAML::VERSION >= '4' + def test_load_psych_4 + # Until we figure out a proper strategy, only `YAML.unsafe_load_file` + # is cached with Psych >= 4 + Help.set_file('a.yml', "foo: &foo\n bar: 42\nplop:\n <<: *foo", 100) + assert_raises FakeYaml::Fallback do + FakeYaml.load_file('a.yml') + end + end + else + def test_load_file + Help.set_file('a.yml', "---\nfoo: bar", 100) + assert_equal({'foo' => 'bar'}, FakeYaml.load_file('a.yml')) + end + + def test_load_file_aliases + Help.set_file('a.yml', "foo: &foo\n bar: 42\nplop:\n <<: *foo", 100) + assert_equal({"foo" => { "bar" => 42 }, "plop" => { "bar" => 42} }, FakeYaml.load_file('a.yml')) + end + + def test_load_file_symbolize_names + Help.set_file('a.yml', "---\nfoo: bar", 100) + FakeYaml.load_file('a.yml') + + if ::Bootsnap::CompileCache::YAML.supported_options.include?(:symbolize_names) + 2.times do + assert_equal({foo: 'bar'}, FakeYaml.load_file('a.yml', symbolize_names: true)) + end + else + assert_raises(FakeYaml::Fallback) do # would call super + FakeYaml.load_file('a.yml', symbolize_names: true) + end + end + end + + def test_load_file_freeze + Help.set_file('a.yml', "---\nfoo", 100) + FakeYaml.load_file('a.yml') + + if ::Bootsnap::CompileCache::YAML.supported_options.include?(:freeze) + 2.times do + string = FakeYaml.load_file('a.yml', freeze: true) + assert_equal("foo", string) + assert_predicate(string, :frozen?) + end + else + assert_raises(FakeYaml::Fallback) do # would call super + FakeYaml.load_file('a.yml', freeze: true) + end + end + end + + def test_load_file_unknown_option + Help.set_file('a.yml', "---\nfoo", 100) + FakeYaml.load_file('a.yml') + + assert_raises(FakeYaml::Fallback) do # would call super + FakeYaml.load_file('a.yml', fallback: true) + end + end + end + + if YAML.respond_to?(:unsafe_load_file) + def test_unsafe_load_file + Help.set_file('a.yml', "foo: &foo\n bar: 42\nplop:\n <<: *foo", 100) + assert_equal({"foo" => { "bar" => 42 }, "plop" => { "bar" => 42} }, FakeYaml.unsafe_load_file('a.yml')) + end + end +end diff -Nru ruby-bootsnap-1.4.6/test/compile_cache_handler_errors_test.rb ruby-bootsnap-1.9.3/test/compile_cache_handler_errors_test.rb --- ruby-bootsnap-1.4.6/test/compile_cache_handler_errors_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/compile_cache_handler_errors_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -19,7 +19,7 @@ def test_input_to_storage_invalid_instance_of_expected_type path = Help.set_file('a.rb', 'a = 3', 100) Bootsnap::CompileCache::ISeq.expects(:input_to_storage).returns('broken') - Bootsnap::CompileCache::ISeq.expects(:input_to_output).with('a = 3').returns('whatever') + Bootsnap::CompileCache::ISeq.expects(:input_to_output).with('a = 3', nil).returns('whatever') _, err = capture_subprocess_io do load(path) end diff -Nru ruby-bootsnap-1.4.6/test/compile_cache_key_format_test.rb ruby-bootsnap-1.9.3/test/compile_cache_key_format_test.rb --- ruby-bootsnap-1.4.6/test/compile_cache_key_format_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/compile_cache_key_format_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -5,6 +5,7 @@ require('fileutils') class CompileCacheKeyFormatTest < Minitest::Test + FILE = File.expand_path(__FILE__) include(TmpdirHelper) R = { @@ -18,16 +19,16 @@ } def test_key_version - key = cache_key_for_file(__FILE__) - exp = [2].pack("L") + key = cache_key_for_file(FILE) + exp = [3].pack("L") assert_equal(exp, key[R[:version]]) end def test_key_compile_option_stable - k1 = cache_key_for_file(__FILE__) - k2 = cache_key_for_file(__FILE__) + k1 = cache_key_for_file(FILE) + k2 = cache_key_for_file(FILE) RubyVM::InstructionSequence.compile_option = { tailcall_optimization: true } - k3 = cache_key_for_file(__FILE__) + k3 = cache_key_for_file(FILE) assert_equal(k1[R[:compile_option]], k2[R[:compile_option]]) refute_equal(k1[R[:compile_option]], k3[R[:compile_option]]) ensure @@ -35,7 +36,7 @@ end def test_key_ruby_revision - key = cache_key_for_file(__FILE__) + key = cache_key_for_file(FILE) exp = if RUBY_REVISION.is_a?(String) [Help.fnv1a_64(RUBY_REVISION) >> 32].pack("L") else @@ -45,39 +46,49 @@ end def test_key_size - key = cache_key_for_file(__FILE__) - exp = File.size(__FILE__) + key = cache_key_for_file(FILE) + exp = File.size(FILE) act = key[R[:size]].unpack("Q")[0] assert_equal(exp, act) end def test_key_mtime - key = cache_key_for_file(__FILE__) - exp = File.mtime(__FILE__).to_i + key = cache_key_for_file(FILE) + exp = File.mtime(FILE).to_i act = key[R[:mtime]].unpack("Q")[0] assert_equal(exp, act) end def test_fetch - actual = Bootsnap::CompileCache::Native.fetch(@tmp_dir, '/dev/null', TestHandler) - assert_equal('NEATO /DEV/NULL', actual) - data = File.read("#{@tmp_dir}/8c/d2d180bbd995df") - assert_equal("neato /dev/null", data.force_encoding(Encoding::BINARY)[64..-1]) - actual = Bootsnap::CompileCache::Native.fetch(@tmp_dir, '/dev/null', TestHandler) - assert_equal('NEATO /DEV/NULL', actual) + if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ + target = 'NUL' + expected_file = "#{@tmp_dir}/36/9eba19c29ffe00" + else + target = '/dev/null' + expected_file = "#{@tmp_dir}/8c/d2d180bbd995df" + end + + actual = Bootsnap::CompileCache::Native.fetch(@tmp_dir, target, TestHandler, nil) + assert_equal("NEATO #{target.upcase}", actual) + + data = File.read(expected_file) + assert_equal("neato #{target}", data.force_encoding(Encoding::BINARY)[64..-1]) + + actual = Bootsnap::CompileCache::Native.fetch(@tmp_dir, target, TestHandler, nil) + assert_equal("NEATO #{target.upcase}", actual) end def test_unexistent_fetch assert_raises(Errno::ENOENT) do - Bootsnap::CompileCache::Native.fetch(@tmp_dir, '123', Bootsnap::CompileCache::ISeq) + Bootsnap::CompileCache::Native.fetch(@tmp_dir, '123', Bootsnap::CompileCache::ISeq, nil) end end private def cache_key_for_file(file) - Bootsnap::CompileCache::Native.fetch(@tmp_dir, file, TestHandler) - data = File.read(Help.cache_path(@tmp_dir, file)) - Help.binary(data[0..31]) + Bootsnap::CompileCache::Native.fetch(@tmp_dir, file, TestHandler, nil) + data = File.binread(Help.cache_path(@tmp_dir, file)) + data.byteslice(0..31) end end diff -Nru ruby-bootsnap-1.4.6/test/compile_cache_test.rb ruby-bootsnap-1.9.3/test/compile_cache_test.rb --- ruby-bootsnap-1.4.6/test/compile_cache_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/compile_cache_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -24,11 +24,24 @@ end def test_no_write_permission_to_cache - path = Help.set_file('a.rb', 'a = 3', 100) - folder = File.dirname(Help.cache_path(@tmp_dir, path)) - FileUtils.mkdir_p(folder) - FileUtils.chmod(0400, folder) - assert_raises(Bootsnap::CompileCache::PermissionError) { load(path) } + if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ + # Always pass this test on Windows because directories aren't read, only + # listed. You can restrict the ability to list directory contents on + # Windows or you can set ACLS on a folder such that it is not allowed to + # list contents. + # + # Since we can't read directories on windows, this specific test doesn't + # make sense. In addtion we test read-only files in + # `test_can_open_read_only_cache` so we are covered testing reading + # read-only files. + pass + else + path = Help.set_file('a.rb', 'a = 3', 100) + folder = File.dirname(Help.cache_path(@tmp_dir, path)) + FileUtils.mkdir_p(folder) + FileUtils.chmod(0400, folder) + load(path) + end end def test_can_open_read_only_cache @@ -106,4 +119,46 @@ load(path) assert(File.size(cp) > 32) # cache was overwritten end + + def test_instrumentation_hit + path = Help.set_file('a.rb', 'a = a = 3', 100) + load(path) + + calls = [] + Bootsnap.instrumentation = ->(event, path) { calls << [event, path] } + + load(path) + + assert_equal [], calls + ensure + Bootsnap.instrumentation = nil + end + + def test_instrumentation_miss + path = Help.set_file('a.rb', 'a = a = 3', 100) + + calls = [] + Bootsnap.instrumentation = ->(event, path) { calls << [event, path] } + + load(path) + + assert_equal [[:miss, 'a.rb']], calls + ensure + Bootsnap.instrumentation = nil + end + + def test_instrumentation_stale + path = Help.set_file('a.rb', 'a = a = 3', 100) + load(path) + path = Help.set_file('a.rb', 'a = a = 4', 101) + + calls = [] + Bootsnap.instrumentation = ->(event, path) { calls << [event, path] } + + load(path) + + assert_equal [[:stale, 'a.rb']], calls + ensure + Bootsnap.instrumentation = nil + end end diff -Nru ruby-bootsnap-1.4.6/test/load_path_cache/cache_test.rb ruby-bootsnap-1.9.3/test/load_path_cache/cache_test.rb --- ruby-bootsnap-1.4.6/test/load_path_cache/cache_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/load_path_cache/cache_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -16,6 +16,7 @@ FileUtils.touch("#{@dir1}/dl#{DLEXT}") FileUtils.touch("#{@dir1}/both.rb") FileUtils.touch("#{@dir1}/both#{DLEXT}") + FileUtils.touch("#{@dir1}/béé.rb") end def teardown @@ -48,17 +49,25 @@ po = [@dir1] cache = Cache.new(NullCache, po) assert_equal("#{@dir1}/a.rb", cache.find('a')) + refute(cache.find('a', try_extensions: false)) cache.push_paths(po, @dir2) assert_equal("#{@dir2}/b.rb", cache.find('b')) + refute(cache.find('b', try_extensions: false)) end def test_extension_append_for_relative_paths po = [@dir1] cache = Cache.new(NullCache, po) + dir1_basename = File.basename(@dir1) Dir.chdir(@dir1) do assert_equal("#{@dir1}/a.rb", cache.find('./a')) + assert_equal("#{@dir1}/a", cache.find('./a', try_extensions: false)) + assert_equal("#{@dir1}/a.rb", cache.find("../#{dir1_basename}/a")) + assert_equal("#{@dir1}/a", cache.find("../#{dir1_basename}/a", try_extensions: false)) assert_equal("#{@dir1}/dl#{DLEXT}", cache.find('./dl')) + assert_equal("#{@dir1}/dl", cache.find('./dl', try_extensions: false)) assert_equal("#{@dir1}/enoent", cache.find('./enoent')) + assert_equal("#{@dir1}/enoent", cache.find('./enoent', try_extensions: false)) end end @@ -66,16 +75,20 @@ po = [@dir1] cache = Cache.new(NullCache, po) assert_equal("#{@dir1}/conflict.rb", cache.find('conflict')) + assert_equal("#{@dir1}/conflict.rb", cache.find('conflict.rb', try_extensions: false)) cache.unshift_paths(po, @dir2) assert_equal("#{@dir2}/conflict.rb", cache.find('conflict')) + assert_equal("#{@dir2}/conflict.rb", cache.find('conflict.rb', try_extensions: false)) end def test_pushed_paths_have_lower_precedence po = [@dir1] cache = Cache.new(NullCache, po) assert_equal("#{@dir1}/conflict.rb", cache.find('conflict')) + assert_equal("#{@dir1}/conflict.rb", cache.find('conflict.rb', try_extensions: false)) cache.push_paths(po, @dir2) assert_equal("#{@dir1}/conflict.rb", cache.find('conflict')) + assert_equal("#{@dir1}/conflict.rb", cache.find('conflict.rb', try_extensions: false)) end def test_directory_caching @@ -88,10 +101,14 @@ def test_extension_permutations cache = Cache.new(NullCache, [@dir1]) assert_equal("#{@dir1}/dl#{DLEXT}", cache.find('dl')) + refute(cache.find('dl', try_extensions: false)) assert_equal("#{@dir1}/dl#{DLEXT}", cache.find("dl#{DLEXT}")) assert_equal("#{@dir1}/both.rb", cache.find("both")) + refute(cache.find("both", try_extensions: false)) assert_equal("#{@dir1}/both.rb", cache.find("both.rb")) + assert_equal("#{@dir1}/both.rb", cache.find("both.rb", try_extensions: false)) assert_equal("#{@dir1}/both#{DLEXT}", cache.find("both#{DLEXT}")) + assert_equal("#{@dir1}/both#{DLEXT}", cache.find("both#{DLEXT}", try_extensions: false)) end def test_relative_paths_rescanned @@ -137,6 +154,36 @@ assert_equal("#{@dir1}/a.rb", cache.find('a')) end + + if RUBY_VERSION >= '2.5' && !(RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/) + # https://github.com/ruby/ruby/pull/4061 + # https://bugs.ruby-lang.org/issues/17517 + OS_ASCII_PATH_ENCODING = RUBY_VERSION >= '3.1' ? Encoding::UTF_8 : Encoding::US_ASCII + + def test_path_encoding + po = [@dir1] + cache = Cache.new(NullCache, po) + + path = cache.find('a') + + assert_equal("#{@dir1}/a.rb", path) + require path + internal_path = $LOADED_FEATURES.last + assert_equal(OS_ASCII_PATH_ENCODING, internal_path.encoding) + assert_equal(OS_ASCII_PATH_ENCODING, path.encoding) + File.write(path, '') + assert_same path, internal_path + + utf8_path = cache.find('béé') + require utf8_path + internal_utf8_path = $LOADED_FEATURES.last + assert_equal("#{@dir1}/béé.rb", utf8_path) + assert_equal(Encoding::UTF_8, internal_utf8_path.encoding) + assert_equal(Encoding::UTF_8, utf8_path.encoding) + File.write(utf8_path, '') + assert_same utf8_path, internal_utf8_path + end + end end end end diff -Nru ruby-bootsnap-1.4.6/test/load_path_cache/change_observer_test.rb ruby-bootsnap-1.9.3/test/load_path_cache/change_observer_test.rb --- ruby-bootsnap-1.4.6/test/load_path_cache/change_observer_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/load_path_cache/change_observer_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -17,18 +17,29 @@ @observer.expects(:push_paths).with(@arr, 'b', 'c') @arr.push('b', 'c') - @observer.expects(:unshift_paths).with(@arr, 'd', 'e') - @arr.unshift('d', 'e') + @observer.expects(:push_paths).with(@arr, 'd', 'e') + @arr.append('d', 'e') - @observer.expects(:push_paths).with(@arr, 'f', 'g') - @arr.concat(%w(f g)) + @observer.expects(:unshift_paths).with(@arr, 'f', 'g') + @arr.unshift('f', 'g') + + @observer.expects(:push_paths).with(@arr, 'h', 'i') + @arr.concat(%w(h i)) + + @observer.expects(:unshift_paths).with(@arr, 'j', 'k') + @arr.prepend('j', 'k') + end + + def test_reinitializes_on_aggressive_modifications + @observer.expects(:push_paths).with(@arr, 'a', 'b', 'c') + @arr.push('a', 'b', 'c') @observer.expects(:reinitialize).times(4) @arr.delete(3) @arr.compact! @arr.map!(&:upcase) - assert_equal('G', @arr.pop) - assert_equal(%w(D E A B C F), @arr) + assert_equal('C', @arr.pop) + assert_equal(%w(A B), @arr) end def test_register_frozen diff -Nru ruby-bootsnap-1.4.6/test/load_path_cache/core_ext/kernel_require_test.rb ruby-bootsnap-1.9.3/test/load_path_cache/core_ext/kernel_require_test.rb --- ruby-bootsnap-1.4.6/test/load_path_cache/core_ext/kernel_require_test.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/load_path_cache/core_ext/kernel_require_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,34 @@ +# frozen_string_literal: true +require('test_helper') + +module Bootsnap + module KernelRequireTest + class KernelLoadTest < MiniTest::Test + def setup + @initial_dir = Dir.pwd + @dir1 = File.realpath(Dir.mktmpdir) + FileUtils.touch("#{@dir1}/a.rb") + FileUtils.touch("#{@dir1}/no_ext") + @dir2 = File.realpath(Dir.mktmpdir) + File.open("#{@dir2}/loads.rb", "wb") { |f| f.write("load 'subdir/loaded'\nload './subdir/loaded'\n") } + FileUtils.mkdir("#{@dir2}/subdir") + FileUtils.touch("#{@dir2}/subdir/loaded") + $LOAD_PATH.push(@dir1) + end + + def teardown + $LOAD_PATH.pop + Dir.chdir(@initial_dir) + FileUtils.rm_rf(@dir1) + FileUtils.rm_rf(@dir2) + end + + def test_no_exstensions_for_kernel_load + assert_raises(LoadError) { load 'a' } + assert(load 'no_ext') + Dir.chdir(@dir2) + assert(load 'loads.rb') + end + end + end +end diff -Nru ruby-bootsnap-1.4.6/test/load_path_cache/path_test.rb ruby-bootsnap-1.9.3/test/load_path_cache/path_test.rb --- ruby-bootsnap-1.4.6/test/load_path_cache/path_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/load_path_cache/path_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -11,15 +11,16 @@ def test_stability require('time') - time_file = Time.method(:rfc2822).source_location[0] - volatile = Path.new(__FILE__) - stable = Path.new(time_file) - unknown = Path.new('/who/knows') - lib = Path.new(RbConfig::CONFIG['libdir'] + '/a') - site = Path.new(RbConfig::CONFIG['sitedir'] + '/b') - bundler = Path.new('/bp/3') + time_file = Time.method(:rfc2822).source_location[0] + volatile = Path.new(__FILE__) + stable = Path.new(time_file) + unknown = Path.new('/who/knows') + lib = Path.new(RbConfig::CONFIG['libdir'] + '/a') + site = Path.new(RbConfig::CONFIG['sitedir'] + '/b') + absolute_prefix = RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ ? ENV['SystemDrive'] : '' + bundler = Path.new(absolute_prefix + '/bp/3') - Bundler.stubs(:bundle_path).returns('/bp') + Bundler.stubs(:bundle_path).returns(absolute_prefix + '/bp') assert(stable.stable?, "The stable path #{stable.path.inspect} was unexpectedly not stable.") refute(stable.volatile?, "The stable path #{stable.path.inspect} was unexpectedly volatile.") @@ -34,10 +35,18 @@ end def test_non_directory? - refute(Path.new('/dev').non_directory?) - refute(Path.new('/nope').non_directory?) - assert(Path.new('/dev/null').non_directory?) - assert(Path.new('/etc/hosts').non_directory?) + if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ + refute(Path.new('c:/dev').non_directory?) + refute(Path.new('c:/nope').non_directory?) + # there isn't a direct analog i could think of + # assert(Path.new('/dev/null').non_directory?) + assert(Path.new("#{ENV['WinDir']}/System32/Drivers/Etc/hosts").non_directory?) + else + refute(Path.new('/dev').non_directory?) + refute(Path.new('/nope').non_directory?) + assert(Path.new('/dev/null').non_directory?) + assert(Path.new('/etc/hosts').non_directory?) + end end def test_volatile_cache_valid_when_mtime_has_not_changed diff -Nru ruby-bootsnap-1.4.6/test/load_path_cache/realpath_cache_test.rb ruby-bootsnap-1.9.3/test/load_path_cache/realpath_cache_test.rb --- ruby-bootsnap-1.4.6/test/load_path_cache/realpath_cache_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/load_path_cache/realpath_cache_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -16,13 +16,13 @@ @symlinked_dir = "#{@base_dir}/symlink" FileUtils.ln_s(@absolute_dir, @symlinked_dir) - real_caller = File.new("#{@absolute_dir}/real_caller.rb", 'w').path + real_caller = File.new("#{@absolute_dir}/real_caller.rb", 'w').tap(&:close).path symlinked_caller = "#{@absolute_dir}/symlinked_caller.rb" FileUtils.ln_s(real_caller, symlinked_caller) EXTENSIONS.each do |ext| - real_required = File.new("#{@absolute_dir}/real_required#{ext}", 'w').path + real_required = File.new("#{@absolute_dir}/real_required#{ext}", 'w').tap(&:close).path symlinked_required = "#{@absolute_dir}/symlinked_required#{ext}" FileUtils.ln_s(real_required, symlinked_required) diff -Nru ruby-bootsnap-1.4.6/test/load_path_cache/store_test.rb ruby-bootsnap-1.9.3/test/load_path_cache/store_test.rb --- ruby-bootsnap-1.4.6/test/load_path_cache/store_test.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/load_path_cache/store_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -68,12 +68,18 @@ def test_retry_on_collision retries = sequence('retries') - File.expects(:binwrite).in_sequence(retries).raises(Errno::EEXIST.new("File exists @ rb_sysopen")) - File.expects(:binwrite).in_sequence(retries).returns(1) + MessagePack.expects(:dump).in_sequence(retries).raises(Errno::EEXIST.new("File exists @ rb_sysopen")) + MessagePack.expects(:dump).in_sequence(retries).returns(1) FileUtils.expects(:mv).in_sequence(retries) store.transaction { store.set('a', 1) } end + + def test_ignore_read_only_filesystem + MessagePack.expects(:dump).raises(Errno::EROFS.new("Read-only file system @ rb_sysopen")) + store.transaction { store.set('a', 1) } + refute(File.exist?(@path)) + end end end end diff -Nru ruby-bootsnap-1.4.6/test/setup_test.rb ruby-bootsnap-1.9.3/test/setup_test.rb --- ruby-bootsnap-1.4.6/test/setup_test.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/setup_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,97 @@ +# frozen_string_literal: true +require('test_helper') + +module Bootsnap + class SetupTest < Minitest::Test + def setup + @_old_env = ENV.to_h + @tmp_dir = Dir.mktmpdir('bootsnap-test') + ENV['BOOTSNAP_CACHE_DIR'] = @tmp_dir + end + + def teardown + ENV.replace(@_old_env) + end + + def test_default_setup + Bootsnap.expects(:setup).with( + cache_dir: @tmp_dir, + development_mode: true, + load_path_cache: true, + compile_cache_iseq: Bootsnap.iseq_cache_supported?, + compile_cache_yaml: true, + compile_cache_json: true, + ) + + Bootsnap.default_setup + end + + def test_default_setup_with_ENV_not_dev + ENV['ENV'] = 'something' + + Bootsnap.expects(:setup).with( + cache_dir: @tmp_dir, + development_mode: false, + load_path_cache: true, + compile_cache_iseq: Bootsnap.iseq_cache_supported?, + compile_cache_yaml: true, + compile_cache_json: true, + ) + + Bootsnap.default_setup + end + + def test_default_setup_with_DISABLE_BOOTSNAP_LOAD_PATH_CACHE + ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'] = 'something' + + Bootsnap.expects(:setup).with( + cache_dir: @tmp_dir, + development_mode: true, + load_path_cache: false, + compile_cache_iseq: Bootsnap.iseq_cache_supported?, + compile_cache_yaml: true, + compile_cache_json: true, + ) + + Bootsnap.default_setup + end + + def test_default_setup_with_DISABLE_BOOTSNAP_COMPILE_CACHE + ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] = 'something' + + Bootsnap.expects(:setup).with( + cache_dir: @tmp_dir, + development_mode: true, + load_path_cache: true, + compile_cache_iseq: false, + compile_cache_yaml: false, + compile_cache_json: false, + ) + + Bootsnap.default_setup + end + + def test_default_setup_with_DISABLE_BOOTSNAP + ENV['DISABLE_BOOTSNAP'] = 'something' + + Bootsnap.expects(:setup).never + Bootsnap.default_setup + end + + def test_default_setup_with_BOOTSNAP_LOG + ENV['BOOTSNAP_LOG'] = 'something' + + Bootsnap.expects(:setup).with( + cache_dir: @tmp_dir, + development_mode: true, + load_path_cache: true, + compile_cache_iseq: Bootsnap.iseq_cache_supported?, + compile_cache_yaml: true, + compile_cache_json: true, + ) + Bootsnap.expects(:logger=).with($stderr.method(:puts)) + + Bootsnap.default_setup + end + end +end diff -Nru ruby-bootsnap-1.4.6/test/test_helper.rb ruby-bootsnap-1.9.3/test/test_helper.rb --- ruby-bootsnap-1.4.6/test/test_helper.rb 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/test_helper.rb 2021-11-24 16:27:37.000000000 +0000 @@ -1,7 +1,16 @@ # frozen_string_literal: true $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__)) + +if defined? Warning + if Warning.respond_to?(:[]=) + Warning[:deprecated] = true + end +end + require('bundler/setup') require('bootsnap') +require('bootsnap/compile_cache/yaml') +require('bootsnap/compile_cache/json') require('tmpdir') require('fileutils') @@ -9,19 +18,25 @@ require('minitest/autorun') require('mocha/minitest') -cache_dir = File.expand_path('../../tmp/bootsnap-compile-cache', __FILE__) -Bootsnap::CompileCache.setup(cache_dir: cache_dir, iseq: true, yaml: false) +cache_dir = File.expand_path('../../tmp/bootsnap/compile-cache', __FILE__) +Bootsnap::CompileCache.setup(cache_dir: cache_dir, iseq: true, yaml: false, json: false) + +if GC.respond_to?(:verify_compaction_references) + # This method was added in Ruby 3.0.0. Calling it this way asks the GC to + # move objects around, helping to find object movement bugs. + GC.verify_compaction_references(double_heap: true, toward: :empty) +end module TestHandler def self.input_to_storage(_i, p) 'neato ' + p end - def self.storage_to_output(d) + def self.storage_to_output(d, _a) d.upcase end - def self.input_to_output(_d) + def self.input_to_output(_d, _a) raise('but why tho') end end @@ -46,12 +61,13 @@ class Test module Help class << self - def binary(str) - str.force_encoding(Encoding::BINARY) - end + def cache_path(dir, file, args_key = nil) + hash = fnv1a_64(file) + unless args_key.nil? + hash ^= fnv1a_64(args_key) + end - def cache_path(dir, file) - hex = fnv1a_64(file).to_s(16) + hex = hash.to_s(16).rjust(16, '0') "#{dir}/#{hex[0..1]}/#{hex[2..-1]}" end @@ -65,6 +81,7 @@ end def set_file(path, contents, mtime) + FileUtils.mkdir_p(File.dirname(path)) File.write(path, contents) FileUtils.touch(path, mtime: mtime) path @@ -82,6 +99,8 @@ Dir.chdir(@tmp_dir) @prev = Bootsnap::CompileCache::ISeq.cache_dir Bootsnap::CompileCache::ISeq.cache_dir = @tmp_dir + Bootsnap::CompileCache::YAML.cache_dir = @tmp_dir + Bootsnap::CompileCache::JSON.cache_dir = @tmp_dir end def teardown @@ -89,5 +108,7 @@ Dir.chdir(@prev_dir) FileUtils.remove_entry(@tmp_dir) Bootsnap::CompileCache::ISeq.cache_dir = @prev + Bootsnap::CompileCache::YAML.cache_dir = @prev + Bootsnap::CompileCache::JSON.cache_dir = @prev end end diff -Nru ruby-bootsnap-1.4.6/test/worker_pool_test.rb ruby-bootsnap-1.9.3/test/worker_pool_test.rb --- ruby-bootsnap-1.4.6/test/worker_pool_test.rb 1970-01-01 00:00:00.000000000 +0000 +++ ruby-bootsnap-1.9.3/test/worker_pool_test.rb 2021-11-24 16:27:37.000000000 +0000 @@ -0,0 +1,22 @@ +# frozen_string_literal: true +require('test_helper') +require('bootsnap/cli') + +module Bootsnap + class WorkerPoolTestTest < Minitest::Test + def test_dispatch + @pool = CLI::WorkerPool.create(size: 2, jobs: { touch: ->(path) { File.write(path, $$.to_s) } }) + @pool.spawn + + Dir.mktmpdir('bootsnap-test') do |tmpdir| + 10.times do |i| + @pool.push(:touch, File.join(tmpdir, i.to_s)) + end + + @pool.shutdown + files = Dir.chdir(tmpdir) { Dir['*'] }.sort + assert_equal 10.times.map(&:to_s), files + end + end + end +end diff -Nru ruby-bootsnap-1.4.6/.travis.yml ruby-bootsnap-1.9.3/.travis.yml --- ruby-bootsnap-1.4.6/.travis.yml 2020-02-24 20:39:36.000000000 +0000 +++ ruby-bootsnap-1.9.3/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,21 +0,0 @@ -language: ruby -sudo: false - -os: - - linux - - osx - -rvm: - - ruby-2.4 - - ruby-2.5 - - ruby-head - -matrix: - allow_failures: - - rvm: ruby-head - include: - - rvm: jruby - os: linux - env: MINIMAL_SUPPORT=1 - -script: bin/ci