diff -Nru ruby-asset-sync-2.9.0/CHANGELOG.md ruby-asset-sync-2.11.0/CHANGELOG.md --- ruby-asset-sync-2.9.0/CHANGELOG.md 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/CHANGELOG.md 2020-03-13 01:38:59.000000000 +0000 @@ -18,11 +18,35 @@ - Nothing +## [2.11.0] - 2020-03-13 + +### Added + +- Add option `remote_file_list_cache_file_path` to skip scanning remote + (https://github.com/AssetSync/asset_sync/pull/400) + + +## [2.10.0] - 2020-02-26 + +### Added + +- Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files + (https://github.com/AssetSync/asset_sync/pull/398) + + +## [2.9.1] - 2020-02-20 + +### Fixed + +- Fix uploading of sprockets manifest file + (https://github.com/AssetSync/asset_sync/pull/397) + + ## [2.9.0] - 2020-01-15 ### Added -- Add option `concurrent_uploads` to improve speed of uploading +- Add option `concurrent_uploads` to improve speed of uploading (https://github.com/AssetSync/asset_sync/pull/393) @@ -938,7 +962,10 @@ * Merge branch 'sinatra' -[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...HEAD +[Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.11.0...HEAD +[2.11.0]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...v2.11.0 +[2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0 +[2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1 [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0 [2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2 [2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1 diff -Nru ruby-asset-sync-2.9.0/debian/changelog ruby-asset-sync-2.11.0/debian/changelog --- ruby-asset-sync-2.9.0/debian/changelog 2020-02-04 14:37:51.000000000 +0000 +++ ruby-asset-sync-2.11.0/debian/changelog 2020-09-12 19:17:54.000000000 +0000 @@ -1,3 +1,22 @@ +ruby-asset-sync (2.11.0-1) unstable; urgency=medium + + * Team Upload. + [ Debian Janitor ] + * Set field Upstream-Contact in debian/copyright. + * Remove obsolete fields Contact, Name from d/u/metadata + (already present in machine-readable debian/copyright). + + [ Cédric Boutillier ] + * [ci skip] Update team name + * [ci skip] Add .gitattributes to keep unwanted files out of the source + package + + [ Nilesh Patra ] + * New upstream version 2.11.0 + * debhelper-compat 13 + + -- Nilesh Patra Sun, 13 Sep 2020 00:47:54 +0530 + ruby-asset-sync (2.9.0-1) unstable; urgency=medium * Team upload diff -Nru ruby-asset-sync-2.9.0/debian/control ruby-asset-sync-2.11.0/debian/control --- ruby-asset-sync-2.9.0/debian/control 2020-02-04 14:37:51.000000000 +0000 +++ ruby-asset-sync-2.11.0/debian/control 2020-09-12 19:17:54.000000000 +0000 @@ -1,10 +1,10 @@ Source: ruby-asset-sync -Maintainer: Debian Ruby Extras Maintainers +Maintainer: Debian Ruby Team Uploaders: Pirate Praveen Section: ruby Testsuite: autopkgtest-pkg-ruby Priority: optional -Build-Depends: debhelper-compat (= 12), +Build-Depends: debhelper-compat (= 13), gem2deb, rake, ruby-activemodel (>= 2:4.1~), diff -Nru ruby-asset-sync-2.9.0/debian/copyright ruby-asset-sync-2.11.0/debian/copyright --- ruby-asset-sync-2.9.0/debian/copyright 2020-02-04 14:37:51.000000000 +0000 +++ ruby-asset-sync-2.11.0/debian/copyright 2020-09-12 19:17:54.000000000 +0000 @@ -1,6 +1,7 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: asset_sync Source: http://rubygems.org/gems/asset_sync +Upstream-Contact: https://github.com/rumblelabs/asset_sync/issues Files: * Copyright: 2011-2013 Rumble Labs Ltd. diff -Nru ruby-asset-sync-2.9.0/debian/patches/disable-aws-tests.patch ruby-asset-sync-2.11.0/debian/patches/disable-aws-tests.patch --- ruby-asset-sync-2.9.0/debian/patches/disable-aws-tests.patch 2020-02-04 14:37:51.000000000 +0000 +++ ruby-asset-sync-2.11.0/debian/patches/disable-aws-tests.patch 2020-09-12 19:17:54.000000000 +0000 @@ -2,6 +2,7 @@ It needs internet access and an aws key to run. Author: Praveen Arimbrathodiyil Last-Update: 2013-06-06 +Forwarded: Not-Needed --- a/spec/integration/aws_integration_spec.rb +++ b/spec/integration/aws_integration_spec.rb diff -Nru ruby-asset-sync-2.9.0/debian/upstream/metadata ruby-asset-sync-2.11.0/debian/upstream/metadata --- ruby-asset-sync-2.9.0/debian/upstream/metadata 2020-02-04 14:37:51.000000000 +0000 +++ ruby-asset-sync-2.11.0/debian/upstream/metadata 2020-09-12 19:17:54.000000000 +0000 @@ -3,7 +3,5 @@ Bug-Database: https://github.com/rumblelabs/asset_sync/issues Bug-Submit: https://github.com/rumblelabs/asset_sync/issues Changelog: https://github.com/rumblelabs/asset_sync/tags -Contact: https://github.com/rumblelabs/asset_sync/issues -Name: asset_sync Repository: https://github.com/rumblelabs/asset_sync.git Repository-Browse: https://github.com/rumblelabs/asset_sync diff -Nru ruby-asset-sync-2.9.0/gemfiles/rails_6_0.gemfile ruby-asset-sync-2.11.0/gemfiles/rails_6_0.gemfile --- ruby-asset-sync-2.9.0/gemfiles/rails_6_0.gemfile 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/gemfiles/rails_6_0.gemfile 2020-03-13 01:38:59.000000000 +0000 @@ -5,6 +5,6 @@ gem "rcov", platforms: :mri_18, group: [:development, :test] gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false gem "jruby-openssl", platform: :jruby -gem "rails", "~> 6.0.0.beta1" +gem "rails", "~> 6.0.0" gemspec path: "../" diff -Nru ruby-asset-sync-2.9.0/lib/asset_sync/config.rb ruby-asset-sync-2.11.0/lib/asset_sync/config.rb --- ruby-asset-sync-2.9.0/lib/asset_sync/config.rb 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/lib/asset_sync/config.rb 2020-03-13 01:38:59.000000000 +0000 @@ -27,7 +27,8 @@ attr_accessor :cache_asset_regexps attr_accessor :include_manifest attr_accessor :concurrent_uploads - attr_writer :public_path + attr_accessor :concurrent_uploads_max_threads + attr_accessor :remote_file_list_cache_file_path # FOG configuration attr_accessor :fog_provider # Currently Supported ['AWS', 'Rackspace'] @@ -87,6 +88,8 @@ self.cache_asset_regexps = [] self.include_manifest = false self.concurrent_uploads = false + self.concurrent_uploads_max_threads = 10 + self.remote_file_list_cache_file_path = nil @additional_local_file_paths_procs = [] load_yml! if defined?(::Rails) && yml_exists? @@ -175,6 +178,19 @@ @public_path || ::Rails.public_path end + def public_path=(path) + # Generate absolute path even when relative path passed in + # Required for generating relative sprockets manifest path + pathname = Pathname(path) + @public_path = if pathname.absolute? + pathname + elsif defined?(::Rails.root) + ::Rails.root.join(pathname) + else + Pathname(::Dir.pwd).join(pathname) + end + end + def load_yml! self.enabled = yml["enabled"] if yml.has_key?('enabled') self.fog_provider = yml["fog_provider"] @@ -210,6 +226,8 @@ self.cache_asset_regexps = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps") self.include_manifest = yml['include_manifest'] if yml.has_key?("include_manifest") self.concurrent_uploads = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads') + self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads') + self.remote_file_list_cache_file_path = yml['remote_file_list_cache_file_path'] if yml.has_key?('remote_file_list_cache_file_path') self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name") self.azure_storage_access_key = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key") diff -Nru ruby-asset-sync-2.9.0/lib/asset_sync/engine.rb ruby-asset-sync-2.11.0/lib/asset_sync/engine.rb --- ruby-asset-sync-2.9.0/lib/asset_sync/engine.rb 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/lib/asset_sync/engine.rb 2020-03-13 01:38:59.000000000 +0000 @@ -43,6 +43,7 @@ config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST') config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST') config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS') + config.remote_file_list_cache_file_path = ENV['ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH'] if ENV.has_key?('ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH') end config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX') diff -Nru ruby-asset-sync-2.9.0/lib/asset_sync/storage.rb ruby-asset-sync-2.11.0/lib/asset_sync/storage.rb --- ruby-asset-sync-2.9.0/lib/asset_sync/storage.rb 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/lib/asset_sync/storage.rb 2020-03-13 01:38:59.000000000 +0000 @@ -37,6 +37,10 @@ self.config.public_path end + def remote_file_list_cache_file_path + self.config.remote_file_list_cache_file_path + end + def ignored_files expand_file_names(self.config.ignored_files) end @@ -58,6 +62,32 @@ (get_local_files + config.additional_local_file_paths).uniq end + def remote_files + return [] if ignore_existing_remote_files? + return @remote_files if @remote_files + + if remote_file_list_cache_file_path && File.file?(remote_file_list_cache_file_path) + begin + content = File.read(remote_file_list_cache_file_path) + return @remote_files = JSON.parse(content) + rescue JSON::ParserError + warn "Failed to parse #{remote_file_list_cache_file_path} as json" + end + end + + @remote_files = get_remote_files + end + + def update_remote_file_list_cache(local_files_to_upload) + return unless remote_file_list_cache_file_path + return if ignore_existing_remote_files? + + File.open(self.remote_file_list_cache_file_path, 'w') do |file| + uploaded = local_files_to_upload + remote_files + file.write(uploaded.to_json) + end + end + def always_upload_files expand_file_names(self.config.always_upload) + get_manifest_path end @@ -243,24 +273,30 @@ end def upload_files - # get a fresh list of remote files - remote_files = ignore_existing_remote_files? ? [] : get_remote_files # fixes: https://github.com/rumblelabs/asset_sync/issues/19 local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq + # Only files. + local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" } if self.config.concurrent_uploads - threads = ThreadGroup.new + jobs = Queue.new + local_files_to_upload.each { |f| jobs.push(f) } + jobs.close + + num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min # Upload new files - local_files_to_upload.each do |f| - next unless File.file? "#{path}/#{f}" # Only files. - threads.add(Thread.new { upload_file f }) + workers = Array.new(num_threads) do + Thread.new do + while f = jobs.pop + upload_file(f) + end + end end - sleep 1 while threads.list.any? # wait for threads to finish uploading + workers.map(&:join) else # Upload new files local_files_to_upload.each do |f| - next unless File.file? "#{path}/#{f}" # Only files. upload_file f end end @@ -271,6 +307,8 @@ data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate) log "Invalidation id: #{data.body["Id"]}" end + + update_remote_file_list_cache(local_files_to_upload) end def sync diff -Nru ruby-asset-sync-2.9.0/lib/asset_sync/version.rb ruby-asset-sync-2.11.0/lib/asset_sync/version.rb --- ruby-asset-sync-2.9.0/lib/asset_sync/version.rb 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/lib/asset_sync/version.rb 2020-03-13 01:38:59.000000000 +0000 @@ -1,5 +1,5 @@ # frozen_string_literal: true module AssetSync - VERSION = "2.9.0" + VERSION = "2.11.0" end diff -Nru ruby-asset-sync-2.9.0/lib/generators/asset_sync/templates/asset_sync.rb ruby-asset-sync-2.11.0/lib/generators/asset_sync/templates/asset_sync.rb --- ruby-asset-sync-2.9.0/lib/generators/asset_sync/templates/asset_sync.rb 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/lib/generators/asset_sync/templates/asset_sync.rb 2020-03-13 01:38:59.000000000 +0000 @@ -66,6 +66,9 @@ # Upload files concurrently # config.concurrent_uploads = false # + # Path to cache file to skip scanning remote + # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json' + # # Fail silently. Useful for environments such as Heroku # config.fail_silently = true # diff -Nru ruby-asset-sync-2.9.0/README.md ruby-asset-sync-2.11.0/README.md --- ruby-asset-sync-2.9.0/README.md 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/README.md 2020-03-13 01:38:59.000000000 +0000 @@ -244,6 +244,12 @@ # # Upload files concurrently # config.concurrent_uploads = false + # + # Number of threads when concurrent_uploads is enabled + # config.concurrent_uploads_max_threads = 10 + # + # Path to cache file to skip scanning remote + # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json' # # Fail silently. Useful for environments such as Heroku # config.fail_silently = true @@ -342,6 +348,8 @@ * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'` * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'` * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'` +* **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10` +* **remote_file_list_cache_file_path**: if present, use this path to cache remote file list to skip scanning remote **default:** `nil` * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled) * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]` * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server. Only rails compiled regexp is matched internally **default**: `[]` @@ -540,6 +548,10 @@ config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] config.prefix = 'assets' + # Can be a `Pathname` or `String` + # Will be converted into an `Pathname` + # If relative, will be converted into an absolute path + # via `::Rails.root` or `::Dir.pwd` config.public_path = Pathname('./public') end ``` diff -Nru ruby-asset-sync-2.9.0/spec/unit/railsless_spec.rb ruby-asset-sync-2.11.0/spec/unit/railsless_spec.rb --- ruby-asset-sync-2.9.0/spec/unit/railsless_spec.rb 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/spec/unit/railsless_spec.rb 2020-03-13 01:38:59.000000000 +0000 @@ -14,7 +14,7 @@ config.fog_region = 'eu-west-1' config.existing_remote_files = "keep" config.prefix = "assets" - config.public_path = Pathname("./public") + config.public_path = "./public" end end @@ -22,8 +22,9 @@ expect(AssetSync.config.prefix).to eq("assets") end - it "should have prefix of assets" do - expect(AssetSync.config.public_path.to_s).to eq("./public") + it "should have public_path" do + expect(AssetSync.config.public_path.to_s).to be_end_with("/public") + expect(AssetSync.config.public_path).to be_absolute end it "should default AssetSync to enabled" do diff -Nru ruby-asset-sync-2.9.0/spec/unit/storage_spec.rb ruby-asset-sync-2.11.0/spec/unit/storage_spec.rb --- ruby-asset-sync-2.9.0/spec/unit/storage_spec.rb 2020-01-15 09:12:53.000000000 +0000 +++ ruby-asset-sync-2.11.0/spec/unit/storage_spec.rb 2020-03-13 01:38:59.000000000 +0000 @@ -70,6 +70,71 @@ storage.upload_files end + it 'should allow custom number of threads' do + @config.concurrent_uploads = true + @config.concurrent_uploads_max_threads = 2 + storage = AssetSync::Storage.new(@config) + + allow(storage).to receive(:get_local_files).and_return(@local_files) + allow(storage).to receive(:get_remote_files).and_return(@remote_files) + allow(File).to receive(:file?).and_return(true) # Pretend they all exist + + expect(Thread).to receive(:new).exactly(2).times.and_call_original + (@local_files - @remote_files + storage.always_upload_files).each do |file| + expect(storage).to receive(:upload_file).with(file) + end + + storage.upload_files + end + + it 'should allow remote_file_list_cache_file_path configuration' do + file_path = './foo.json' + @config.remote_file_list_cache_file_path = file_path + storage = AssetSync::Storage.new(@config) + + allow(storage).to receive(:get_local_files).and_return(@local_files) + File.write(file_path, @remote_files.to_json) + expect(storage).not_to receive(:get_remote_files) + allow(File).to receive(:file?).and_return(true) # Pretend they all exist + + (@local_files - @remote_files + storage.always_upload_files).each do |file| + expect(storage).to receive(:upload_file).with(file) + end + + expect(storage).not_to receive(:warn) + storage.upload_files + + # update remote_file_list_cache corretly + updated = JSON.parse(File.read(file_path)) + expect(updated.sort.uniq).to eq (@remote_files + @local_files + storage.always_upload_files).sort.uniq + + File.delete(file_path) + end + + it 'should work with broken cache' do + file_path = './foo.json' + @config.remote_file_list_cache_file_path = file_path + + storage = AssetSync::Storage.new(@config) + + File.write(file_path, 'some non-json text file content') + + allow(storage).to receive(:get_local_files).and_return(@local_files) + allow(storage).to receive(:get_remote_files).and_return(@remote_files) + allow(File).to receive(:file?).and_return(true) # Pretend they all exist + + (@local_files - @remote_files + storage.always_upload_files).each do |file| + expect(storage).to receive(:upload_file).with(file) + end + + # when broken, warning message should be prompted + expect(storage).to receive(:warn) + + storage.upload_files + + File.delete(file_path) + end + it 'should upload updated non-fingerprinted files' do @local_files = [ 'public/image.png',