From 3977cd901a39eaf2ad8263f8c702459787ca8934 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 06:34:29 +0000 Subject: [PATCH] Modernize the bundled plugins and remove the dead integrations The plugin set shipped with the gem was written between 2012 and 2015 and had not been reviewed against the services and libraries it talks to. Each of the forty-five plugins was checked against the current state of its service, its API and its gem, and the set was reduced to what still has a use and made to work on Ruby 3.3 through 4.0. Removed eleven plugins whose service or API no longer exists: the three Twitter plugins, both Pocket plugins, HipChat, Google Calendar, livedoor Weather, So-net G-Guide and Chan-Toru, and Google News link rewriting. Their specs, integration recipes and optional dependencies went with them. A Recipe naming one now fails at load rather than at runtime, and doc/PLUGINS.md section 8 records each reason. Migrated PublishAmazonS3 and the S3 path of StoreFile from AWS SDK for Ruby v1 to aws-sdk-s3, keeping the Recipe keys and adding an optional region; both now also accept the SDK's own credential chain, so a bucket can be reached without a secret in the Recipe. Dropped xml-simple from CustomFeedSVNLog in favour of REXML, which the framework already depends on, and nkf from FilterDescriptionLink, whose parser detects an encoding for itself. PublishHatenaBookmark remains Needs rework -- the service and its API are current, the WSSE interface it speaks is not -- but no longer sends a password digest over plain HTTP. Corrected defects the review turned up: `interval` was never waited, because the line meant to do it assigned to a local variable named sleep; FilterDescriptionLink read neither of its settings, because it tested for Hash and a Recipe produces a Hashie::Mash; FilterAbsoluteURI treated an https link as relative; PublishMemcached could not take a port written as a number; FilterTumblrResize and FilterImage did not recognise the URL shapes those services now use. Added Automatic::Http, the one way in for what plugins fetch, so that the scheme allowlist, timeouts and redirect limit are decided once rather than by omission in seven plugins. A link in a pipeline item comes from a feed, and URI.open on such a string would read a local file as readily as an article. Every Supported plugin now has deterministic local tests that reach no network and need no credential, and a spec holds doc/PLUGINS.md and README to the files in plugins/, so a catalogue entry or a count cannot drift. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PpGaCj366pPvFCa3JX4Dh5 --- Gemfile | 14 +- README.md | 52 +-- automatic.gemspec | 8 +- config/feed2console.yml | 14 +- doc/BASIC_DESIGN.md | 35 +- doc/DEPLOYMENT.md | 63 +-- doc/PLUGINS.md | 375 ++++++++++-------- doc/POLICY.md | 108 +++-- doc/REQUIREMENTS.md | 17 +- doc/VERSIONS | 2 + lib/automatic.rb | 5 +- lib/automatic/feed_parser.rb | 11 +- lib/automatic/http.rb | 104 +++++ plugins/custom_feed/svn_log.rb | 95 +++-- plugins/filter/absolute_uri.rb | 57 ++- plugins/filter/accept.rb | 70 ++-- plugins/filter/clear.rb | 8 +- plugins/filter/description_link.rb | 83 ++-- plugins/filter/full_feed.rb | 97 ++--- plugins/filter/github_feed.rb | 51 ++- plugins/filter/google_news.rb | 51 --- plugins/filter/ignore.rb | 65 ++- plugins/filter/image.rb | 49 ++- plugins/filter/image_source.rb | 96 ++--- plugins/filter/one.rb | 32 +- plugins/filter/rand.rb | 20 +- plugins/filter/sanitize.rb | 49 +-- plugins/filter/sort.rb | 35 +- plugins/filter/tumblr_resize.rb | 42 +- plugins/notify/ikachan.rb | 123 +++--- plugins/provide/fluentd.rb | 53 ++- plugins/publish/amazon_s3.rb | 101 +++-- plugins/publish/console.rb | 23 +- plugins/publish/console_link.rb | 25 +- plugins/publish/eject.rb | 61 ++- plugins/publish/fluentd.rb | 66 +-- plugins/publish/google_calendar.rb | 86 ---- plugins/publish/hatena_bookmark.rb | 147 ++++--- plugins/publish/hipchat.rb | 48 --- plugins/publish/instapaper.rb | 115 +++--- plugins/publish/memcached.rb | 50 +-- plugins/publish/pocket.rb | 47 --- plugins/publish/twitter.rb | 60 --- plugins/store/database.rb | 74 ++-- plugins/store/file.rb | 156 +++++--- plugins/store/full_text.rb | 43 +- plugins/store/permalink.rb | 26 +- plugins/subscription/chan_toru.rb | 59 --- plugins/subscription/feed.rb | 43 +- plugins/subscription/g_guide.rb | 60 --- plugins/subscription/link.rb | 51 ++- plugins/subscription/pocket.rb | 52 --- plugins/subscription/text.rb | 65 ++- plugins/subscription/tumblr.rb | 87 ++-- plugins/subscription/twitter.rb | 71 ---- plugins/subscription/twitter_search.rb | 51 --- plugins/subscription/weather.rb | 34 -- plugins/subscription/xml.rb | 62 +-- script/build | 11 +- spec/doc/plugins_catalogue_spec.rb | 127 ++++++ spec/lib/automatic/http_spec.rb | 79 ++++ spec/plugins/custom_feed/svn_log_spec.rb | 111 +++++- spec/plugins/filter/absolute_uri_spec.rb | 22 +- spec/plugins/filter/description_link_spec.rb | 32 +- spec/plugins/filter/github_feed_spec.rb | 14 +- spec/plugins/filter/google_news_spec.rb | 71 ---- spec/plugins/filter/image_source_spec.rb | 63 ++- spec/plugins/filter/image_spec.rb | 56 ++- spec/plugins/filter/one_spec.rb | 12 +- spec/plugins/filter/rand_spec.rb | 4 +- spec/plugins/filter/sanitize_spec.rb | 30 +- spec/plugins/filter/tumblr_resize_spec.rb | 29 ++ spec/plugins/notify/ikachan_spec.rb | 117 ++++-- spec/plugins/provide/fluentd_spec.rb | 91 +++-- spec/plugins/publish/amazon_s3_spec.rb | 105 +++-- spec/plugins/publish/eject_spec.rb | 33 +- spec/plugins/publish/fluentd_spec.rb | 76 ++-- spec/plugins/publish/google_calendar_spec.rb | 91 ----- spec/plugins/publish/hatena_bookmark_spec.rb | 185 ++++----- spec/plugins/publish/hipchat_spec.rb | 76 ---- spec/plugins/publish/instapaper_spec.rb | 122 +++--- spec/plugins/publish/memcached_spec.rb | 99 +++-- spec/plugins/publish/pocket_spec.rb | 58 --- spec/plugins/publish/twitter_spec.rb | 80 ---- spec/plugins/store/file_spec.rb | 73 +++- spec/plugins/subscription/chan_toru_spec.rb | 58 --- spec/plugins/subscription/feed_spec.rb | 24 +- spec/plugins/subscription/g_guide_spec.rb | 84 ---- spec/plugins/subscription/link_spec.rb | 2 +- spec/plugins/subscription/pocket_spec.rb | 64 --- spec/plugins/subscription/tumblr_spec.rb | 49 ++- .../subscription/twitter_search_spec.rb | 64 --- spec/plugins/subscription/twitter_spec.rb | 80 ---- spec/plugins/subscription/weather_spec.rb | 51 --- spec/plugins/subscription/xml_spec.rb | 2 +- spec/spec_helper.rb | 2 +- test/integration/test_add_pocket.yml | 26 -- test/integration/test_chan_toru.yml | 21 - test/integration/test_google_news.yml | 21 - test/integration/test_pocket.yml | 22 - test/integration/test_twitter_search.yml | 22 - test/integration/test_weather.yml | 19 - 102 files changed, 2849 insertions(+), 3276 deletions(-) create mode 100644 lib/automatic/http.rb delete mode 100644 plugins/filter/google_news.rb delete mode 100644 plugins/publish/google_calendar.rb delete mode 100644 plugins/publish/hipchat.rb delete mode 100644 plugins/publish/pocket.rb delete mode 100644 plugins/publish/twitter.rb delete mode 100644 plugins/subscription/chan_toru.rb delete mode 100644 plugins/subscription/g_guide.rb delete mode 100644 plugins/subscription/pocket.rb delete mode 100644 plugins/subscription/twitter.rb delete mode 100644 plugins/subscription/twitter_search.rb delete mode 100644 plugins/subscription/weather.rb create mode 100644 spec/doc/plugins_catalogue_spec.rb create mode 100644 spec/lib/automatic/http_spec.rb delete mode 100644 spec/plugins/filter/google_news_spec.rb delete mode 100644 spec/plugins/publish/google_calendar_spec.rb delete mode 100644 spec/plugins/publish/hipchat_spec.rb delete mode 100644 spec/plugins/publish/pocket_spec.rb delete mode 100644 spec/plugins/publish/twitter_spec.rb delete mode 100644 spec/plugins/subscription/chan_toru_spec.rb delete mode 100644 spec/plugins/subscription/g_guide_spec.rb delete mode 100644 spec/plugins/subscription/pocket_spec.rb delete mode 100644 spec/plugins/subscription/twitter_search_spec.rb delete mode 100644 spec/plugins/subscription/twitter_spec.rb delete mode 100644 spec/plugins/subscription/weather_spec.rb delete mode 100644 test/integration/test_add_pocket.yml delete mode 100644 test/integration/test_chan_toru.yml delete mode 100644 test/integration/test_google_news.yml delete mode 100644 test/integration/test_pocket.yml delete mode 100644 test/integration/test_twitter_search.yml delete mode 100644 test/integration/test_weather.yml diff --git a/Gemfile b/Gemfile index 1f5a83b..5fed213 100644 --- a/Gemfile +++ b/Gemfile @@ -54,11 +54,6 @@ group :plugins, :sanitize, optional: true do gem 'sanitize' end -# FilterDescriptionLink, which normalizes a fetched page's encoding with it. -group :plugins, :nkf, optional: true do - gem 'nkf' -end - # The autodiscovery and inspect subcommands. No plugin and no Recipe uses it. group :plugins, :autodiscovery, optional: true do gem 'feedbag', '>= 1.0', '< 2.0' @@ -77,10 +72,9 @@ group :fluentd, optional: true do gem 'fluent-logger' # PublishFluentd and ProvideFluentd, with a Fluentd instance end -group :svn_log, optional: true do - gem 'xml-simple' # CustomFeedSVNLog, with the svn command +group :s3, optional: true do + gem 'aws-sdk-s3' # PublishAmazonS3 and the s3 path of StoreFile, with a bucket end -# PublishAmazonS3 and the s3n:// path of StoreFile call AWS::S3, which only -# AWS SDK for Ruby v1 provided. No currently published gem satisfies them, so -# there is no group to select; they need rework. See doc/PLUGINS.md. +# CustomFeedSVNLog needs the svn command and no gem: it reads `svn log --xml` +# with REXML, which is a runtime dependency of the framework already. diff --git a/README.md b/README.md index a3b748b..ddb7361 100644 --- a/README.md +++ b/README.md @@ -93,15 +93,16 @@ second user. The project began in February 2012 and this is the first release since 2015. The core, the Recipe format and the plugin contract are unchanged; what has changed -is that it runs on a current Ruby, installs from a current RubyGems, and says -plainly which of its plugins still work. See [`doc/VERSIONS`](doc/VERSIONS). +is that it runs on a current Ruby, installs from a current RubyGems, and ships +a plugin set every part of which still has somewhere to talk to. See +[`doc/VERSIONS`](doc/VERSIONS). ## 2. Features - **Recipes in YAML.** A job is a file, not a program. No Ruby is written to wire a pipeline together. -- **Forty-five plugins** across seven categories: subscribe, custom feed, - filter, store, provide, notify, publish. +- **34 plugins** across seven categories: subscribe, custom feed, filter, + store, provide, notify, publish — and every one of them has a current use. - **Markdown out of the box.** `PublishMarkdown` writes the result as a plain Markdown document, to a file or to standard output, with no service and no credential behind it. It is the natural end of a new Recipe. @@ -118,9 +119,10 @@ plainly which of its plugins still work. See [`doc/VERSIONS`](doc/VERSIONS). the framework: `gem install automatic` brings four pure-Ruby gems and the command, and installs neither an HTML parser nor a database — let alone an AWS SDK. -- **Honest about what is broken.** Every plugin is classified, with its reason, - in [`doc/PLUGINS.md`](doc/PLUGINS.md). Nothing dead is stubbed into looking - alive. +- **No museum.** Every plugin is classified, with its reason, in + [`doc/PLUGINS.md`](doc/PLUGINS.md). Nothing dead is stubbed into looking + alive, and an integration whose service has gone is removed rather than + kept as a fossil. ## 3. Architecture @@ -367,26 +369,29 @@ like a shipped plugin replaces it. ### Which plugins still work -Forty-five plugins ship with the gem, most of them written between 2012 and -2015. Several talk to services that have since shut down. Every one is -classified in [`doc/PLUGINS.md`](doc/PLUGINS.md) section 6, with its settings -and the reason for its status: +34 plugins ship with the gem. Every one is classified in +[`doc/PLUGINS.md`](doc/PLUGINS.md) section 6, with its settings and the reason +for its status: | Status | Count | Meaning | | --- | --- | --- | | **Supported** | 23 | Works on the supported Rubies with current dependencies | -| **Supported (external)** | 9 | Works, but needs something you provide: a service, a command, a data file | -| **Needs rework** | 3 | The service exists; this plugin speaks a replaced interface | -| **Unsupported** | 10 | The service has shut down | +| **Supported (external)** | 10 | Works, but needs something you provide: a service, a command, a credential, a data file | +| **Needs rework** | 1 | The service exists; this plugin speaks a replaced interface | -Restoring one of the three in **Needs rework** — `FilterGoogleNews`, -`PublishAmazonS3`, `PublishHatenaBookmark` — is self-contained work and a good -first contribution. +Eleven plugins were removed in this release rather than kept as history: each +talked to a service that has shut down, or through an API that has been +withdrawn with no replacement. They are listed with their reasons in +[`doc/PLUGINS.md`](doc/PLUGINS.md) section 8, and Git history holds the code. +A Recipe naming one of them now fails at load, before anything runs. + +Restoring the one in **Needs rework** — `PublishHatenaBookmark` — is +self-contained work and a good first contribution. No plugin here is stubbed, mocked or simulated to make a test pass. Where a plugin's gem is not installed its spec is skipped and says which gem is missing; where the plugin still loads, its spec covers what does not need the -service. A dead integration is never made to look alive. +service. A dead integration is never made to look alive — it is removed. The contract, a worked example, and how to test a plugin are in [`doc/PLUGINS.md`](doc/PLUGINS.md) sections 3 and 4. @@ -523,7 +528,7 @@ COVERAGE=on bundle exec rake spec longer exists is never stubbed into passing. - `test/integration/` holds Recipes for exercising plugins against real services. They are run by hand, are not part of the suite, and are never run - in CI. Most need a credential, a dead service, or both — read one before + in CI. Most need a credential or a service you run — read one before running it. The required check installs the bundle, builds the gem, loads the library, runs @@ -547,7 +552,7 @@ bundle exec rake bundle exec bin/automatic -c config/feed2console.yml ``` -Contributions are welcome — a new plugin, or reviving one of the three that need +Contributions are welcome — a new plugin, or reviving the one that needs rework, most of all. 1. Fork the repository. @@ -565,8 +570,8 @@ Two rules worth knowing before you start: - **A gem needed by one plugin is not a dependency of the framework.** Require it at the top of the plugin's own file. -- **Nothing dead is faked.** A plugin whose service has shut down is classified - as unsupported, not stubbed into passing a test. +- **Nothing dead is faked.** A plugin whose service has shut down is removed, + not stubbed into passing a test. - Repository: - Issues: @@ -586,6 +591,7 @@ Two rules worth knowing before you start: │ ├── log.rb Levelled logging to standard output │ ├── feed_maker.rb Builds pipeline values from plain data │ ├── feed_parser.rb Fetches and parses feeds +│ ├── http.rb The one way in for what plugins fetch │ ├── opml.rb OPML parser, for the opmlparser subcommand │ ├── environment.rb Bundler setup for a source checkout │ └── version.rb @@ -619,7 +625,7 @@ this repository. No document here defers to another repository. | [`doc/PLUGIN_DEVELOPMENT.md`](doc/PLUGIN_DEVELOPMENT.md) | A complete user plugin and practical testing guidance | | [`doc/REQUIREMENTS.md`](doc/REQUIREMENTS.md) | What the system is for, what it guarantees, where its responsibility ends | | [`doc/BASIC_DESIGN.md`](doc/BASIC_DESIGN.md) | How it is composed: the parts, their responsibilities, the flow of a run | -| [`doc/PLUGINS.md`](doc/PLUGINS.md) | The Recipe format, the plugin contract, and the catalogue of all 45 plugins | +| [`doc/PLUGINS.md`](doc/PLUGINS.md) | The Recipe format, the plugin contract, and the catalogue of every shipped plugin | | [`doc/POLICY.md`](doc/POLICY.md) | How a change is made and judged: style, dependencies, tests, versioning | | [`doc/DEPLOYMENT.md`](doc/DEPLOYMENT.md) | Installing, scheduling, operating, and what to do when it fails | | [`doc/RELEASING.md`](doc/RELEASING.md) | For maintainers: building, verifying and publishing the gem | diff --git a/automatic.gemspec b/automatic.gemspec index 62a67ad..2df8ea7 100644 --- a/automatic.gemspec +++ b/automatic.gemspec @@ -98,10 +98,10 @@ Gem::Specification.new do |spec| # groups, doc/DEPLOYMENT.md and doc/POLICY.md section 9.1. # # rexml and rss left the standard library and became gems over the 3.x - # series, and nkf followed after 3.3. Each gem listed here is listed because - # a file in lib/ requires it, and a library's move out of the standard - # library is not by itself a reason to declare it: nkf is a plugin's - # dependency and is in the Gemfile's optional groups instead. + # series. Each gem listed here is listed because a file in lib/ requires it, + # and a library's move out of the standard library is not by itself a reason + # to declare it: a library only a plugin needs goes in the Gemfile's optional + # groups instead. spec.add_dependency 'activesupport', '>= 7.1', '< 9.0' # plugin loader, XML subscription spec.add_dependency 'hashie', '>= 4.0', '< 6.0' # Recipe spec.add_dependency 'rexml', '>= 3.2', '< 4.0' # OPML parser diff --git a/config/feed2console.yml b/config/feed2console.yml index 61a8668..ad4c75f 100644 --- a/config/feed2console.yml +++ b/config/feed2console.yml @@ -1,7 +1,13 @@ +# Print a public feed to the terminal. +# +# automatic -c ~/.automatic/config/example/feed2console.yml +# +# The same pipeline as feed2markdown.yml with a different last plugin, which +# is the plugin to end a Recipe with while you are still writing it. Nothing +# here needs an account, a credential or a gem beyond the ones +# `gem install automatic` brings. + global: - timezone: Asia/Tokyo - cache: - base: /tmp log: level: info @@ -9,6 +15,6 @@ plugins: - module: SubscriptionFeed config: feeds: - - http://blog.id774.net/post/feed/ + - https://www.ruby-lang.org/en/feeds/news.rss - module: PublishConsole diff --git a/doc/BASIC_DESIGN.md b/doc/BASIC_DESIGN.md index c658275..ff69ef3 100644 --- a/doc/BASIC_DESIGN.md +++ b/doc/BASIC_DESIGN.md @@ -63,7 +63,7 @@ lib/automatic/recipe.rb lib/automatic/pipeline.rb | | ~/.automatic/plugins//.rb v v lib/automatic/log.rb lib/automatic/feed_maker.rb - | + | lib/automatic/http.rb v standard output ``` @@ -74,8 +74,8 @@ Dependency points downward, and there is no edge back up: - `Automatic::CLI` knows the framework. Nothing in the framework knows the CLI. - `Automatic::Pipeline` knows how to find and call a plugin. It knows no plugin. - A plugin knows `Automatic::Log`, `Automatic::FeedMaker`, - `Automatic::FeedParser` and its own libraries. It knows no other plugin, with - one deliberate exception noted in section 4.9. + `Automatic::FeedParser`, `Automatic::Http` and its own libraries. It knows no + other plugin. - `Automatic::Log` and `Automatic::FeedMaker` are leaves. They depend on nothing in this repository. @@ -226,9 +226,10 @@ It is a module with state rather than an injected object. That is a consequence of plugins calling `Automatic::Log` directly, which keeps a plugin's signature to `(config, pipeline)`. -### 4.8 `lib/automatic/feed_maker.rb` and `feed_parser.rb` +### 4.8 `lib/automatic/feed_maker.rb`, `feed_parser.rb` and `http.rb` -The adapters between "some data" and the pipeline shape. +The adapters between "some data" and the pipeline shape, and the one way in for +what is fetched. - `FeedParser.get_url(url)` fetches a URL and parses it as a feed. - `FeedParser.parse_html(html)` builds a feed whose items are the page's links, @@ -240,8 +241,20 @@ The adapters between "some data" and the pipeline shape. arbitrary payload in `content_encoded`, which is the route by which the XML subscription plugin feeds the Fluentd provide plugin. -Both use Ruby's bundled `rss` library. That is the reason the pipeline value has -the shape it has. +The first two use Ruby's bundled `rss` library. That is the reason the pipeline +value has the shape it has. + +- `Http.read(url)` fetches a URL and returns the body; `Http.uri(url)` returns + a validated URI and `Http.fetchable?(url)` answers whether there is one. + +`Automatic::Http` exists because the decisions a fetch implies — which schemes +are allowed, how long to wait, how many redirects to follow, what to send as a +User-Agent — were being made separately by every plugin that fetched, mostly by +omission. It is a helper of about twenty lines and not a client: a plugin that +wants something else calls Ruby directly. The scheme allowlist is the part that +earns it a file of its own, because a link in a pipeline item comes from a feed +and `URI.open` on such a string will read a local file as readily as an +article. ### 4.9 `plugins/` — `Automatic::Plugin::*` @@ -301,8 +314,8 @@ Its specification — what it writes for each field, how HTML in a body is reduced, where the output goes — is in [`PLUGINS.md`](PLUGINS.md) section 6.7, because it is a plugin's specification and not a property of the design. -Two shared pieces sit inside `plugins/` rather than in `lib/`, because they are -plugin implementation and the framework does not use them: +One shared piece sits inside `plugins/` rather than in `lib/`, because it is +plugin implementation and the framework does not use it: - `plugins/store/database.rb` — the `Automatic::Plugin::Database` mixin: opens the SQLite database named in the Recipe, creates the table from the including @@ -310,10 +323,6 @@ plugin implementation and the framework does not use them: `for_each_new_feed`, which yields only items whose key is not already stored. `StorePermalink` and `StoreFullText` are this mixin plus a model and a column list. -- `plugins/subscription/chan_toru.rb` requires `g_guide.rb` and delegates to it. - This is the one plugin-to-plugin dependency, it is explicit, and it is not a - pattern to copy. - ### 4.10 `db/`, `config/`, `assets/` Fallbacks inside the installation, used when the corresponding part of the user diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index f315326..3c5d191 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -98,8 +98,8 @@ bundle install bundle exec rake ``` -That adds `activerecord`, `sqlite3`, `nokogiri`, `sanitize`, `nkf` and -`feedbag`, and their specs then run as part of the ordinary suite. The setting +That adds `activerecord`, `sqlite3`, `nokogiri`, `sanitize` and `feedbag`, +and their specs then run as part of the ordinary suite. The setting is written to the checkout's own `.bundle/config`, which is not committed; `bundle config unset --local with` returns the checkout to the minimum, and `bundle install` afterwards. @@ -447,8 +447,7 @@ This table is the list. Which plugin needs which gem, how to install it, and whether the plugin still works are all here, and nothing else repeats it. None of these gems is installed by `gem install automatic` or by a default -`bundle install`. Install one only if you use the plugin, and check the status -column first — several of these plugins talk to services that no longer exist. +`bundle install`. Install one only if you use the plugin. **Installed gem**: `gem install `. **Checkout**: `bundle config set --local with ` and `bundle install`, because `bundle exec` sees only the @@ -460,32 +459,35 @@ bundle. `plugins` is every group in the first block at once. | `FilterImageSource`, `FilterDescriptionLink`, `SubscriptionLink`, `SubscriptionTumblr` | `nokogiri` | `gem install nokogiri` | `html` | Supported (`SubscriptionTumblr` external) | | `PublishMarkdown` | `nokogiri`, for HTML bodies only | `gem install nokogiri` | `html` | Supported; runs without it | | `FilterSanitize` | `sanitize` | `gem install sanitize` | `sanitize` | Supported | -| `FilterDescriptionLink` | `nkf`, as well as `nokogiri` | `gem install nkf` | `nkf` | Supported | | `autodiscovery` and `inspect` subcommands | `feedbag` | `gem install feedbag` | `autodiscovery` | Supported | | `FilterFullFeed` | `nokogiri`, and a siteinfo file | `gem install nokogiri` | `html` | Supported (external) | -| `CustomFeedSVNLog` | `xml-simple`, and the `svn` command | `gem install xml-simple` | `svn_log` | Supported (external) | +| `CustomFeedSVNLog` | the `svn` command; no gem | — | — | Supported (external) | | `ProvideFluentd`, `PublishFluentd` | `fluent-logger`, and a Fluentd instance | `gem install fluent-logger` | `fluentd` | Supported (external) | | `PublishMemcached` | `dalli`, and a memcached server | `gem install dalli` | `memcached` | Supported (external) | +| `PublishAmazonS3`, `StoreFile` S3 path | `aws-sdk-s3`, and a bucket | `gem install aws-sdk-s3` | `s3` | Supported (external) | +| `PublishInstapaper` | an Instapaper account; no gem | — | — | Supported (external) | | `PublishEject` | the `eject` or `drutil` command | — | — | Supported (external) | | `NotifyIkachan` | an `ikachan` gateway you run | — | — | Supported (external) | -| `StoreFile`, S3 path only | the `aws-sdk` v1 interface | — | — | Needs rework | -| `PublishAmazonS3` | the `aws-sdk` v1 interface | — | — | Needs rework | -| `PublishTwitter`, `SubscriptionTwitterSearch` | — | — | — | Unsupported | -| `PublishPocket`, `SubscriptionPocket` | — | — | — | Unsupported | -| `PublishHipchat` | — | — | — | Unsupported | -| `PublishGoogleCalendar` | — | — | — | Unsupported | -| `SubscriptionWeather` | — | — | — | Unsupported | - -The `plugins` group is the first six rows: the optional gems of the plugins -whose specs need nothing but the gem. The three rows below it are in their own -groups only, because each also needs a service or a command, and installing a -gem alone would not make the plugin — or its spec — work. - -The two AWS rows are listed for completeness rather than as instructions. -They call `AWS::S3`, which AWS SDK for Ruby version 1 provided and the current -`aws-sdk-s3` does not; installing a gem will not make them work, and the plugins -need rework. `StoreFile` makes that requirement lazily, so its ordinary HTTP -download path works with no AWS gem installed at all. +| `PublishHatenaBookmark` | the current Hatena API, which it does not speak | — | — | Needs rework | + +The `plugins` group is the first five rows: the optional gems of the plugins +whose specs need nothing but the gem. The gems below it are in their own groups +only, because each of those plugins also needs a service, a bucket or a +command, and installing a gem alone would not make the plugin — or its spec — +work. + +Both S3 rows make their requirement lazily, so `StoreFile`'s ordinary HTTP +download path and `PublishAmazonS3` in `mode: test` work with no AWS gem +installed at all. Leaving `access_key` and `secret_key` out of the Recipe is the +better way to use them: the SDK then takes credentials from the environment, a +shared profile or an instance role, and no long-lived secret sits in a file. + +Two gems left this table in v26.08 and are not needed by anything now: +`xml-simple`, which `CustomFeedSVNLog` used to parse `svn log --xml` and which +REXML — already a dependency of the framework — parses instead, and `nkf`, +which `FilterDescriptionLink` used to normalize a page's encoding and which the +HTML parser does for itself. If you installed either for this project, nothing +here wants it any more. No optional group is installed by default and none is installed in required CI, so these plugins are outside what a green build guarantees. Installing a group @@ -523,9 +525,14 @@ not touch this directory. **`The gem is not installed. It is needed by ...`** — a plugin's optional dependency is missing. The message names the gem, the plugin and the command to -install it; the table above says the same thing, and says whether the plugin is -one that no longer works. A bare `LoadError: cannot load such file -- ` is -the same situation from a plugin that is no longer supported. +install it, and the table above says the same thing. + +**`Automatic::NoPluginError: unknown plugin named ...`** — a Recipe names a +plugin that does not ship. Check the spelling against +[`PLUGINS.md`](PLUGINS.md) section 6; if the name is in section 8, the plugin +was removed because the service behind it no longer exists, and the Recipe +needs a different last step rather than a reinstall. Nothing has run when this +is raised. **`unknown plugin named X`** — the Recipe names a module the loader cannot resolve. Check the spelling against [`PLUGINS.md`](PLUGINS.md) section 6, and @@ -562,7 +569,7 @@ are not run in CI: bundle exec bin/automatic -c test/integration/test_sort.yml ``` -Most of them need a credential, a service that no longer exists, or both. Read +Most of them need a credential or a service you run. Read the Recipe before running it, and check the plugin's status in [`PLUGINS.md`](PLUGINS.md) section 6. diff --git a/doc/PLUGINS.md b/doc/PLUGINS.md index 3a83a6d..2b4f033 100644 --- a/doc/PLUGINS.md +++ b/doc/PLUGINS.md @@ -257,7 +257,7 @@ Examples, including the ones that are easy to get wrong: | `FilterAbsoluteURI` | `filter/absolute_uri.rb` | | `CustomFeedSVNLog` | `custom_feed/svn_log.rb` | | `PublishHatenaBookmark` | `publish/hatena_bookmark.rb` | -| `SubscriptionTwitterSearch` | `subscription/twitter_search.rb` | +| `FilterDescriptionLink` | `filter/description_link.rb` | The category directory is not decoration: it is half of the lookup key. A file in a directory whose name is not a prefix of the underscored class name is never @@ -396,7 +396,7 @@ see the optional plugin dependencies in doc/DEPLOYMENT.md. ``` Pass `gem_name:` where the gem's name differs from the path required, as -`xml-simple` does from `xmlsimple`. +`activerecord` does from `active_record`. That is what keeps a gem needed by one plugin out of everyone else's installation. A gem used by a single plugin is not added to the framework's @@ -409,6 +409,26 @@ support in `StoreFile`, for instance — the `require` goes inside the branch th uses it, so the plugin loads and its ordinary path works without that gem installed. +### 3.8.1 Fetching + +A plugin that fetches over HTTP calls `Automatic::Http`: + +```ruby +body = Automatic::Http.read(url) # the body, or an exception +Automatic::Http.uri(url) # a validated URI, or an exception +Automatic::Http.fetchable?(url) # for skipping an item rather than failing +``` + +It is a helper and not a client: it opens the URL through `open-uri` with the +scheme restricted to HTTP and HTTPS, a connect and a read timeout, a bounded +redirect chain and this project named as the agent. A URL string carrying +characters a URI may not — a space, a Japanese query term — is escaped and +parsed again rather than raising. + +The scheme restriction is the part that matters most: **a link in a pipeline +item comes from a feed, which is to say from outside.** `URI.open` on such a +string will read `file:///etc/passwd` as readily as an article. + ### 3.9 Testing a plugin Construct it, run it, assert on what came back: @@ -520,19 +540,24 @@ Section 6 lists every plugin shipped in the gem. Each carries a status: | Status | Meaning | | --- | --- | | **Supported** | Works on the supported Ruby versions with the current dependencies. Covered by the default test suite where it can be. | -| **Supported (external)** | The plugin is current, but it needs something the operator provides — a running service, an installed command, a data file. | -| **Needs rework** | The service or API still exists, but this plugin speaks an interface that has been replaced. It will not work as written. | -| **Unsupported** | The service has shut down, or its API is no longer reachable in the way this plugin uses it. Kept as history, not expected to work. | +| **Supported (external)** | The plugin is current, but it needs something the operator provides — a running service, an installed command, a credential, a data file. | +| **Needs rework** | The service and the capability still exist, but this plugin speaks an interface that has been replaced. It will not work as written, and restoring it is a self-contained piece of work. | + +There is no fourth row. There used to be one, holding plugins whose service had +shut down, and the plugins that were in it have been removed rather than kept: +see section 8. Two rules govern this table, and they are the reason it exists at all: -- **Nothing is faked.** A plugin in the last two rows is not stubbed, mocked or - simulated to make a test pass or a catalogue entry look better. Its specs are - excluded from the default suite because its gem is absent or its service is - gone, and that absence is the honest signal. -- **Nothing is deleted for being old.** These plugins are the record of what the - framework was used for, and several remain useful as templates for a - replacement. Removal is a separate, deliberate decision. +- **Nothing is faked.** A plugin is not stubbed, mocked or simulated to make a + test pass or a catalogue entry look better. Where a plugin's gem is absent + its specs are excluded from the default suite, and that absence is the honest + signal. A service that no longer answers is not given a fake endpoint to + answer with; the plugin goes. +- **Nothing is kept for being old.** A plugin ships because it has a current + practical use, not because it once did. Git history is where the previous + implementations are, and it keeps them without their being installed on + anyone's machine. **Supported is not the same as covered by the required workflow.** A Supported plugin whose gem is an optional plugin dependency — the store plugins, the ones @@ -546,14 +571,13 @@ never executes. **This classification is a snapshot taken in August 2026,** based on the published status of each service and on what each plugin's code actually calls. -The statuses in the "external service" rows depend on the outside world and can -change without any commit here. Where a status was reached from published -information rather than from a live check, the entry says so. To verify one -yourself, run its Recipe from `test/integration` by hand; those are not part of -CI and never will be. +The statuses that depend on an outside service can change without any commit +here. Where a status was reached from published information rather than from a +live check, the entry says so. To verify one yourself, run its Recipe from +`test/integration` by hand; those are not part of CI and never will be. -Restoring a **Needs rework** plugin is a self-contained piece of work and a good -first contribution. It is not part of this stage of the modernization. +Restoring the one **Needs rework** plugin is a self-contained piece of work and +a good first contribution. --- @@ -586,6 +610,12 @@ with. A feed that fails after its retries is logged and skipped; the others still run. +`interval` is now waited. The line that was meant to wait it assigned to a +local variable named `sleep` and returned at once, in this plugin and in every +other that had a retry loop, so a Recipe asking to be gentle with a host was +not being gentle. A Recipe that set `interval` will take longer than it used +to and will behave as it always said it did. + #### SubscriptionLink — **Supported** `subscription/link.rb`. Fetches pages and makes an item of every ``. @@ -647,53 +677,15 @@ nokogiri`, or the `html` group in a checkout — and it depends on the theme a given blog uses and on Tumblr's page structure. Verify against the blog you mean to follow before putting it in `cron`, and set `interval`. -#### SubscriptionTwitter — **Unsupported** - -`subscription/twitter.rb`. Scraped `twitter.com` by matching CSS class names as -they were in 2014 (`js-tweet-text`, `tweet-timestamp js-permalink js-nav`). The -site is now X, the markup is gone, and a timeline is not served to an -unauthenticated client at all. Nothing in this plugin can match. - -#### SubscriptionTwitterSearch — **Unsupported** - -`subscription/twitter_search.rb`. Calls `Twitter::Client.new(...).search(...)`, -the interface of the `twitter` gem version 4. The gem's current major version -does not have that class, and search on the current API is not available on the -free tier. Restoring it means a new plugin against the current API and a paid -plan, not a dependency bump. - -#### SubscriptionPocket — **Unsupported** - -`subscription/pocket.rb`. Reads the Pocket v3 `retrieve` endpoint through the -`pocket-ruby` gem. Pocket was shut down by Mozilla in July 2025 and the API is -gone. The gem remains on RubyGems; the service does not. - -#### SubscriptionWeather — **Unsupported** - -`subscription/weather.rb`. Uses the `weather_hacker` gem against livedoor -Weather Hacks, which was terminated in 2020. The gem was last published in 2013. - -#### SubscriptionGGuide — **Unsupported** - -`subscription/g_guide.rb`. Searches a Japanese television schedule RSS at -`tv.so-net.ne.jp`, over plain HTTP. The service is no longer operating at that -address. Reported status, not a live check. - -#### SubscriptionChanToru — **Unsupported** - -`subscription/chan_toru.rb`. Runs `SubscriptionGGuide` and rewrites each link to -a So-net "CHAN-TORU" recording URL. It cannot work while `SubscriptionGGuide` -does not, and the target service is likewise gone. This is also the one plugin -that depends on another plugin, which is noted in -[`BASIC_DESIGN.md`](BASIC_DESIGN.md) section 4.9 and is not a pattern to copy. - ### 6.2 CustomFeed #### CustomFeedSVNLog — **Supported (external)** `custom_feed/svn_log.rb`. Runs `svn log --xml` against a repository and makes a -feed of the revisions. Needs the `svn` command and the `xml-simple` gem, neither -of which is installed by this gem. +feed of the revisions. Needs the `svn` command, which is the operator's to +install, and **no gem of its own**: it reads the document with REXML, which is +a runtime dependency of the framework already. It used to need `xml-simple`, +whose last release was in 2021. | Key | Type | Meaning | | --- | --- | --- | @@ -701,8 +693,14 @@ of which is installed by this gem. | `fetch_items` | integer | Revisions to fetch. Default `30`. | | `title` | string | Channel title. Default empty. | -`target` is interpolated into a shell command. Point it at a repository URL you -control and nothing else. +The command is run as an argument vector rather than through a shell, so a +repository URL cannot become part of a command line. Point `target` at a +repository you control regardless: `svn` itself will do what the URL tells it +to. + +A repository with no revisions in the window asked for returns the pipeline +unchanged, with a warning. RSS 1.0 has no representation for a channel with no +items, and this used to end the run with a parser error. ### 6.3 Filter @@ -725,7 +723,9 @@ kept, with a warning. #### FilterAccept — **Supported** `filter/accept.rb`. The complement of `FilterIgnore`: keeps only items that -match. Same three keys, same substring rule. +match. Same three keys, same substring rule. An item whose field is missing is +not matched, and says so; it used to end the run with a `NoMethodError`, which +is not what its complement does with the same item. #### FilterSort — **Supported** @@ -760,19 +760,31 @@ plugin has done the work, so that later plugins publish nothing. No settings. #### FilterImage — **Supported** -`filter/image.rb`. Sets `link` to `nil` unless it ends in `.jpg`, `.jpeg`, -`.gif`, `.png` or `.tiff`. Note that it does not remove the items — it blanks -their links, and the plugins after it skip items whose link is `nil`. No -settings. +`filter/image.rb`. Sets `link` to `nil` unless it names an image. Note that it +does not remove the items — it blanks their links, and the plugins after it +skip items whose link is `nil`. No settings. + +The extensions are `.jpg`, `.jpeg`, `.gif`, `.png`, `.tif`, `.tiff`, `.webp` +and `.avif`, and the test is on the URL's **path**. Both of those changed: +`.webp` and `.avif` are what an image link on the current web frequently is, +and testing the whole URL meant that `photo.jpg?w=1280` — which is how most of +what serves images now serves them — was not recognised as one. A Recipe using +this filter will therefore keep links it used to blank. #### FilterImageSource — **Supported** `filter/image_source.rb`. Replaces each item with one item per image found: the -`` values in the description, or, if there are none, the images on the -page the link points at. Fetching pages means network access. No settings. +images in the description, or, if there are none, the images on the page the +link points at. Fetching pages means network access. No settings. Needs `nokogiri`: `gem install nokogiri`, or the `html` group in a checkout. +The description is read with that parser rather than scanned for the literal +text ` recipe.rb -> pipeline.rb | v | Automatic::Plugin::* v | -log.rb feed_maker.rb feed_parser.rb | +log.rb feed_maker.rb feed_parser.rb http.rb ^----------------------------------+ ``` @@ -115,11 +117,12 @@ Dependency points one way and there is no edge back up: under `lib/automatic/` other than `cli.rb` may reference it. - **`Pipeline` knows how to find and call a plugin; it knows no plugin.** A reference to a plugin class name in the framework is a design error. -- **A plugin knows `Log`, `FeedMaker`, `FeedParser` and its own libraries.** It - does not know another plugin, does not know the CLI, and does not reach into - `Automatic` for directories other than through the helpers provided. -- **`Log`, `FeedMaker` and `FeedParser` are leaves.** They depend on nothing - else in this repository. +- **A plugin knows `Log`, `FeedMaker`, `FeedParser`, `Http` and its own + libraries.** It does not know another plugin, does not know the CLI, and does + not reach into `Automatic` for directories other than through the helpers + provided. +- **`Log`, `FeedMaker`, `FeedParser` and `Http` are leaves.** They depend on + nothing else in this repository. - A new responsibility goes to the part that owns it. Where it appears to belong to two, the boundary is wrong and is corrected, rather than the code being written across it. @@ -134,9 +137,8 @@ Dependency points one way and there is no edge back up: - The framework does not inspect the pipeline between plugins. - A plugin does not modify framework state. It does not set `Automatic.root_dir` or `Automatic.user_dir`, and it does not change the log level. -- Plugin-to-plugin dependency is avoided. The one existing case, - `SubscriptionChanToru` on `SubscriptionGGuide`, is explicit and is not a - pattern to extend. +- Plugin-to-plugin dependency is avoided. A plugin that wants another plugin's + result is a Recipe with two entries in it. - Shared plugin code that the framework does not use stays under `plugins/`, not in `lib/`. `plugins/store/database.rb` is where it is for that reason. @@ -222,8 +224,14 @@ Dependency points one way and there is no edge back up: documentation says to set it. Scraping politely is a requirement, not a courtesy. - A URL that comes from a setting or from feed content is escaped before use, - and is never interpolated into a shell command without the operator having - named it deliberately. + and is never interpolated into a shell command. An external command is run as + an argument vector. +- **A URL that comes from feed content is external input.** It is fetched + through `Automatic::Http`, which restricts the scheme to HTTP and HTTPS, + because `URI.open` on such a string will read a local file as readily as an + article. +- Every request has a connect and a read timeout. An unattended run that hangs + is a failure mode with no upper bound on its cost. ### 1.11 Security and credentials @@ -403,27 +411,59 @@ repository level only; see section 10. --- -## 4. Deprecated and broken plugins +## 4. The life of a plugin Plugins outlive the services they talk to. The policy for what happens then: -- **A plugin is not deleted for being old.** These are the record of what the - framework was used for, and several are usable as templates for a replacement. +- **A shipped plugin has a current practical use.** That is the condition for + being in the gem, and it is a condition that has to keep being met, not one + met once. Having been useful is not the test. - **A plugin is classified, in `PLUGINS.md` section 6**, as Supported, Supported - (external), Needs rework or Unsupported, with the reason. The classification - is the deliverable; the code is left alone. -- **A dead plugin is never faked into life.** No stub of a shut-down service, no - mock that makes an integration look alive, no test that asserts against a - simulation. This is Invariant 7 and it has no exceptions. -- A plugin whose gem cannot be installed on a supported Ruby has its gem removed - from the development dependencies, and its spec then does not load. That - absence is the honest signal and is not worked around. -- Restoring a plugin to a service's current API is a separate change, one plugin - at a time, with the catalogue entry updated in the same commit. -- **Deleting a plugin** requires that its service is gone, that no replacement - is intended, and that the removal is recorded in `VERSIONS` naming the plugin - so that an operator whose Recipe breaks can find out why. It is not done as - part of a general clean-up. + (external) or Needs rework, with the reason. There is no status meaning + "does not work and never will"; a plugin in that position is removed. +- **A dead integration is never faked into life.** No stub of a shut-down + service, no mock that makes an integration look alive, no test that asserts + against a simulation. This is Invariant 7 and it has no exceptions. If a + plugin can only be made to look supported by simulating what it talks to, + what it needs is deletion, not a double. +- **Unsupported code is not kept for preservation.** Git history holds every + implementation this project ever shipped, and holds it without installing it + on anyone's machine or listing it in a catalogue an operator reads for + guidance. A plugin retained only so that its code exists somewhere is + retained for a reason the version control system already covers. +- **The judgement is evidenced.** "Nobody uses that any more" is not a reason. + The service's own site, its API documentation or its published shutdown + notice is, and the reason goes in the catalogue entry or in the removal + record. Where the evidence cannot be obtained, the plugin is classified + **Needs rework** and kept: the failure mode of guessing is deleting something + that works. +- **Needs rework means restoration is realistic.** A capability the service + still offers, reachable by a current API, with the work amounting to a + migration rather than a new project. A plugin whose service is gone is not + Needs rework; it is removed. +- **Restoring a plugin to a service's current API** is a separate change, one + plugin at a time, with the catalogue entry updated in the same commit. Where + the current API forces a credential format the old Recipe cannot express, + that is a breaking change and is documented as one rather than hidden behind + a translation. +- **Removing a plugin removes all of it**: implementation, specs, example and + integration Recipes, catalogue entry, and any optional dependency nothing + else needs. A class left behind to raise "this no longer works" is not a + courtesy; the loader's `NoPluginError` says the same thing earlier and + without shipping code. +- **A removal is recorded in `VERSIONS`** in enough detail that an operator + whose Recipe breaks can find out why, and the reasons are kept in + `PLUGINS.md`. Removals are batched into a release rather than trickled, so + that an upgrade has one list to read. +- **A modernized plugin keeps its Recipe's meaning.** Class names, setting + names and defaults are not changed for tidiness; where a fix makes a plugin + behave as its documentation always said it did, that is still a behaviour + change and the catalogue entry says so. +- **A Supported plugin has deterministic local tests.** They cover this side of + the boundary — settings, request construction, serialization, response + handling, error behaviour — and reach no network, need no credential and + require no running service. The availability of somebody else's API is not + something a unit test can assert and is not something required CI waits on. --- @@ -621,9 +661,9 @@ manager, plugin manifest or resolver is introduced beyond it. operator's bundle, and tight enough to exclude a major version this code has not been checked against. - A gem no longer used by anything committed here is removed. -- A gem whose service no longer exists is removed from the development - dependencies, and the plugin using it is reclassified in `PLUGINS.md` rather - than deleted. +- A gem whose service no longer exists is removed, along with the plugin that + needed it; see section 4. An optional group left with nothing to install is + deleted from the `Gemfile` in the same change. - **Nothing is vendored.** Dependencies come from RubyGems, which keeps their licences theirs. diff --git a/doc/REQUIREMENTS.md b/doc/REQUIREMENTS.md index 2f9542c..8662237 100644 --- a/doc/REQUIREMENTS.md +++ b/doc/REQUIREMENTS.md @@ -357,12 +357,17 @@ The framework reaches nothing. Plugins reach the network, and where they do: - Nothing is fetched on the framework's own initiative: no update check, no telemetry, no phone-home. -Many of the services these plugins were written against have since shut down or -replaced their APIs. Which plugins still work is a fact about the outside world -rather than about this repository, so it is recorded per plugin in -[`PLUGINS.md`](PLUGINS.md) section 6 and kept current there. Two requirements -follow: - +Services shut down and APIs are replaced, and whether a given plugin still works +is a fact about the outside world rather than about this repository. It is +recorded per plugin in [`PLUGINS.md`](PLUGINS.md) section 6 and kept current +there. Three requirements follow: + +- **A shipped plugin shall be practically usable on the supported Ruby + versions.** Shipping it is a statement that it does what its entry says, + given what that entry says the operator must provide. A plugin for which that + statement can no longer be made shall be corrected or removed; it shall not + be carried indefinitely as a record of what once existed, which is what the + version control history is for. - **A plugin that cannot work shall say so** — in the catalogue and, where it runs at all, in its log output. It shall not be quietly left to fail at runtime. diff --git a/doc/VERSIONS b/doc/VERSIONS index fa34225..4e8c559 100644 --- a/doc/VERSIONS +++ b/doc/VERSIONS @@ -9,6 +9,8 @@ v26.08 (Release Date: TBD) - Verify TLS certificates when publishing to Instapaper instead of accepting an unverified connection. - Modernize gem packaging and dependency policy, separating core requirements from optional plugin dependencies so that an installation can be minimal, complete or extended one plugin at a time, and excluding development and generated files. - Classify every shipped plugin by its current support status rather than simulating obsolete services in tests. +- Modernize the bundled plugins for current Ruby, libraries and services, and remove the integrations that are no longer viable. +- Remove the plugins for Twitter, Pocket, HipChat, Google Calendar, livedoor Weather, So-net G-Guide and Chan-Toru, and Google News link rewriting; their services or APIs no longer exist, and a Recipe naming one now fails at load. - Rebuild the test and CI strategy for current RSpec and Ruby, with deterministic isolation from user data and external services. - Add Markdown as the primary service-independent publication format, with a documented and tested first-run workflow. - Rebuild the maintained documentation around current usage, architecture, policy, plugins and deployment, and remove superseded historical documents. diff --git a/lib/automatic.rb b/lib/automatic.rb index 7e5f4d0..a6bd3c3 100755 --- a/lib/automatic.rb +++ b/lib/automatic.rb @@ -17,6 +17,7 @@ module Automatic require 'automatic/log' require 'automatic/recipe' require 'automatic/pipeline' + require 'automatic/http' require 'automatic/feed_parser' require 'automatic/feed_maker' require 'automatic/version' @@ -44,13 +45,13 @@ class << self # its absence into a message naming the gem, what wanted it and how to get # it. Nothing here resolves, installs or tracks a dependency: the `require` # is the plugin's own, made where the plugin makes it, and this only - # replaces `cannot load such file -- nkf` with a sentence an operator can + # replaces `cannot load such file -- dalli` with a sentence an operator can # act on. See doc/POLICY.md section 9.1 and doc/PLUGINS.md section 3.8. # # Automatic.require_optional('sanitize', needed_by: 'FilterSanitize') # # `gem_name` is given where it differs from the path required, as - # `xmlsimple` does from the `xml-simple` gem. + # `active_record` does from the `activerecord` gem. def require_optional(feature, needed_by:, gem_name: feature) require feature rescue LoadError => e diff --git a/lib/automatic/feed_parser.rb b/lib/automatic/feed_parser.rb index 7d10c2e..30e0bb7 100644 --- a/lib/automatic/feed_parser.rb +++ b/lib/automatic/feed_parser.rb @@ -13,20 +13,19 @@ module Automatic module FeedParser - require 'open-uri' require 'rss' - require 'uri' + require 'automatic/http' # Fetch a URL and parse it as a feed. Validation is off, because feeds in # the wild frequently are not valid and are still readable. + # + # Fetching goes through Automatic::Http, which is where the scheme + # allowlist, the timeouts and the redirect limit live. def self.get_url(url) return if url.nil? Automatic::Log.puts('info', "Parsing Feed: #{url}") - feed = URI.parse(url).normalize - feed.open do |http| - RSS::Parser.parse(http.read, false) - end + RSS::Parser.parse(Automatic::Http.read(url), false) end # Build a feed whose items are the links of an HTML document. This is how diff --git a/lib/automatic/http.rb b/lib/automatic/http.rb new file mode 100644 index 0000000..29dc1ad --- /dev/null +++ b/lib/automatic/http.rb @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Http +# Author: id774 (More info: http://id774.net) +# Source Code:: https://github.com/id774/automaticruby +# License:: The GPL version 3, or LGPL version 3 (Dual License). +# Contact:: idnanashi@gmail.com +# Created:: Aug 15, 2026 +# Updated:: Aug 15, 2026 +# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. +# +# One way in for everything the plugins fetch over HTTP. +# +# This exists because seven plugins were each calling `URI.open` on a string, +# and the four decisions that call implies -- which schemes are allowed, how +# long to wait, how many redirects to follow, what to send as a User-Agent -- +# were being made seven times, mostly by omission. They are made once here. +# It is a helper, not a client framework: one method fetches, one builds a URI, +# and a plugin that wants something else still calls Ruby directly. +# See doc/POLICY.md section 9.2 and doc/PLUGINS.md section 3.8. + +require 'open-uri' +require 'uri' +require 'automatic/version' + +module Automatic + module Http + # A link in a pipeline item comes from a feed, which is to say from + # outside. `URI.open` on such a string will happily read `file:///etc/passwd` + # or run an FTP session; a plugin fetching an article body wants neither. + SCHEMES = %w[http https].freeze + + OPEN_TIMEOUT = 10 + READ_TIMEOUT = 30 + + # open-uri follows redirects itself and refuses an HTTPS-to-HTTP downgrade. + # This bounds the chain so that a redirect loop ends as an error rather + # than as an unattended run that never returns. + REDIRECT_LIMIT = 5 + + USER_AGENT = "Automatic Ruby/#{Automatic::VERSION} " \ + '(+https://github.com/id774/automaticruby)' + + # The RFC 2396 parser is named directly: `URI.escape` was removed in Ruby + # 3.0, and `URI::Parser` became the RFC 3986 parser in 3.4, which reports + # #escape as obsolete. This spelling means the same thing on every + # supported Ruby. + ESCAPER = URI::RFC2396_Parser.new + + class << self + # Fetch a URL and return its body as a string. Raises rather than + # returning nil: a plugin's retry handling is built around an exception, + # and a body that could not be fetched is not an empty body. + def read(url) + open(url, &:read) + end + + # Fetch a URL and yield the IO, for a caller that would rather stream + # than hold the whole body. + def open(url, &block) + uri(url).open( + 'User-Agent' => USER_AGENT, + open_timeout: OPEN_TIMEOUT, + read_timeout: READ_TIMEOUT, + redirect: true, + max_redirects: REDIRECT_LIMIT, + &block + ) + end + + # Parse a URL into a URI this framework will fetch, or raise. A string + # carrying characters a URI may not (a space, a Japanese query term) is + # escaped and parsed again, which is what the plugins used to do for + # themselves before every call. + def uri(url) + string = url.to_s.strip + raise ArgumentError, 'no URL to fetch' if string.empty? + + parsed = parse(string) + unless SCHEMES.include?(parsed.scheme) + raise ArgumentError, "not an HTTP or HTTPS URL: #{string}" + end + + parsed.normalize + end + + # Whether a string is a URL this framework will fetch. For a plugin that + # skips an item rather than failing the run on one. + def fetchable?(url) + uri(url) + true + rescue ArgumentError, URI::InvalidURIError + false + end + + private + + def parse(string) + URI.parse(string) + rescue URI::InvalidURIError + URI.parse(ESCAPER.escape(string)) + end + end + end +end diff --git a/plugins/custom_feed/svn_log.rb b/plugins/custom_feed/svn_log.rb index 37ab2ff..23e3d56 100644 --- a/plugins/custom_feed/svn_log.rb +++ b/plugins/custom_feed/svn_log.rb @@ -5,55 +5,92 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 29, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. +require 'rexml/document' require 'rss/maker' +require 'time' module Automatic::Plugin class CustomFeedSVNLog - Automatic.require_optional('xmlsimple', - gem_name: 'xml-simple', - needed_by: 'CustomFeedSVNLog') + DEFAULT_FETCH_ITEMS = 30 - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end def run - revisions = XmlSimple.xml_in(`svn log #{svn_log_argument}`)["logentry"] - @pipeline << RSS::Maker.make("1.0") {|maker| - maker.channel.title = @config["title"] || "" - maker.channel.about = "" - maker.channel.description = "" - maker.channel.link = base_url - revisions.each {|rev| - item = maker.items.new_item - item.title = "#{rev["msg"]} by #{rev["author"]}" - item.link = base_url+"/!svn/bc/#{rev["revision"]}" - item.date = Time.parse(rev["date"][0]) - } - } - return @pipeline + entries = revisions + if entries.empty? + # RSS 1.0 has no representation for a channel with no items, and a + # repository with no revisions in the window asked for is an ordinary + # answer rather than a failure. + Automatic::Log.puts('warn', "No revisions returned by svn log for #{base_url}") + return @pipeline + end + + @pipeline << feed(entries) + @pipeline end private - def base_url - return @config['target'].gsub(/\/$/, "") + def feed(revisions) + RSS::Maker.make('1.0') do |maker| + maker.channel.title = @config['title'].to_s + maker.channel.about = '' + maker.channel.description = '' + maker.channel.link = base_url + + revisions.each do |revision| + item = maker.items.new_item + item.title = "#{revision['msg']} by #{revision['author']}" + item.link = "#{base_url}/!svn/bc/#{revision['revision']}" + item.date = Time.parse(revision['date']) + end + end + end + + # `svn log --xml` as REXML sees it. REXML is a runtime dependency of this + # framework already -- the OPML parser uses it -- so this plugin needs the + # svn command and no gem of its own. + def revisions + REXML::Document.new(svn_log).elements.to_a('/log/logentry').map do |entry| + { + 'revision' => entry.attributes['revision'].to_s, + 'author' => text(entry, 'author'), + 'msg' => text(entry, 'msg'), + 'date' => text(entry, 'date') + } + end end - def svn_log_argument - return [ - base_url, - "--xml", - "--limit=#{limit}" - ].join(" ") + def text(entry, name) + element = entry.elements[name] + element.nil? ? '' : element.text.to_s + end + + # The command is run as an argument vector rather than through a shell, so + # a repository URL cannot become part of a command line. Point `target` at + # a repository you control regardless: svn itself will do what the URL + # tells it to. + def svn_log + output = IO.popen(['svn', 'log', base_url, '--xml', "--limit=#{limit}"], + err: File::NULL, &:read) + raise "svn log failed for #{base_url}" unless $?.success? + + output + end + + def base_url + @base_url ||= @config['target'].to_s.sub(%r{/\z}, '') end def limit - return @config['fetch_items'] || 30 + value = @config['fetch_items'].to_i + value.positive? ? value : DEFAULT_FETCH_ITEMS end end end diff --git a/plugins/filter/absolute_uri.rb b/plugins/filter/absolute_uri.rb index 1b3fa9d..b7fa05e 100644 --- a/plugins/filter/absolute_uri.rb +++ b/plugins/filter/absolute_uri.rb @@ -5,43 +5,56 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 20, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterAbsoluteURI require 'uri' - def initialize(config, pipeline=[]) - @config = config + # Anything already carrying a scheme is left alone. The earlier spelling + # of this test matched `http://` only, so an https link was treated as + # relative and had the base prepended to it. + ABSOLUTE = %r{\A[a-zA-Z][a-zA-Z0-9+.\-]*://} + + # URI::Parser became the RFC 3986 parser in Ruby 3.4, which reports #escape + # as obsolete. The RFC 2396 parser is what this was always reaching and is + # spelled the same way on every supported Ruby. + ESCAPER = URI::RFC2396_Parser.new + + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline + @base = base_url end def run - @return_feeds = [] - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - feed.link = rewrite(feed.link) unless feed.link.nil? - } - @return_feeds << feeds + @pipeline.each_with_object([]) do |feeds, returned| + next if feeds.nil? + + feeds.items.each do |item| + item.link = rewrite(item.link) unless item.link.nil? end - } - @return_feeds + returned << feeds + end end private - def rewrite(string) - if /^http:\/\/.*$/ =~ string - return string - end - if /[^\/]$/ =~ @config['url'] - @config['url'] = @config['url'] + '/' - end - string = @config['url'] + string.sub(/^\./,'').sub(/^\//,'') - string = URI::RFC2396_Parser.new.escape(string) - return string + # Read once, in the constructor: the earlier version appended the trailing + # slash to the Recipe's own config mapping, which is the plugin's input + # rather than its state. + def base_url + url = @config['url'].to_s + return url if url.empty? || url.end_with?('/') + + "#{url}/" + end + + def rewrite(link) + return link if ABSOLUTE.match?(link) + + ESCAPER.escape(@base + link.sub(/\A\./, '').sub(%r{\A/}, '')) end end end diff --git a/plugins/filter/accept.rb b/plugins/filter/accept.rb index 3d1b76d..cd691cb 100644 --- a/plugins/filter/accept.rb +++ b/plugins/filter/accept.rb @@ -5,59 +5,51 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 4, 2013 -# Updated:: Feb 21, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterAccept - def initialize(config, pipeline=[]) - @config = config + FIELDS = %i[title link description].freeze + + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # The complement of FilterIgnore: keeps only the items that match. Matching + # is a substring test, so an empty keyword matches everything. def run - @return_feeds = [] - @pipeline.each {|feeds| - new_feeds = [] - unless feeds.nil? - feeds.items.each {|items| - new_feeds << items if contain(items) == true - } - end - @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0 - } - @return_feeds + @pipeline.each_with_object([]) do |feeds, returned| + kept = feeds.nil? ? [] : feeds.items.select { |item| contain?(item) } + returned << Automatic::FeedMaker.create_pipeline(kept) unless kept.empty? + end end private - def contain(items) - detection = false - unless @config['title'].nil? - @config['title'].each {|e| - if items.title.include?(e.chomp) - detection = true - Automatic::Log.puts("info", "Contain by title: #{items.link}") - end - } - end - unless @config['link'].nil? - @config['link'].each {|e| - if items.link.include?(e.chomp) - detection = true - Automatic::Log.puts("info", "Contain by link: #{items.link}") - end - } + def contain?(item) + FIELDS.any? do |field| + Array(@config[field.to_s]).any? do |keyword| + matched?(item, field, keyword.to_s.chomp) + end end - unless @config['description'].nil? - @config['description'].each {|e| - if items.description.include?(e.chomp) - detection = true - Automatic::Log.puts("info", "Contain by description: #{items.link}") - end - } + end + + # An item whose field is missing is not matched, and says so. The earlier + # version called #include? on it and ended the run with a NoMethodError, + # which is not what the complementary filter does with the same item. + def matched?(item, field, keyword) + value = item.send(field) + unless value.respond_to?(:include?) + Automatic::Log.puts('warn', "Invalid feed detected in accept process with #{field}") + return false end - detection + + return false unless value.include?(keyword) + + Automatic::Log.puts('info', "Contain by #{field}: #{item.link}") + true end end end diff --git a/plugins/filter/clear.rb b/plugins/filter/clear.rb index 4b3acb8..792da4e 100644 --- a/plugins/filter/clear.rb +++ b/plugins/filter/clear.rb @@ -5,16 +5,18 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Oct 20, 2014 -# Updated:: Oct 20, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterClear - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Returns an empty pipeline, to end a Recipe after a store plugin has done + # the work so that later plugins publish nothing. def run [] end diff --git a/plugins/filter/description_link.rb b/plugins/filter/description_link.rb index 2ee7b5d..75cd339 100644 --- a/plugins/filter/description_link.rb +++ b/plugins/filter/description_link.rb @@ -5,15 +5,12 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Oct 03, 2014 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterDescriptionLink - require 'erb' - Automatic.require_optional('nkf', needed_by: 'FilterDescriptionLink') Automatic.require_optional('nokogiri', needed_by: 'FilterDescriptionLink') - require 'open-uri' require 'uri' # URI.extract and URI::PATTERN answer through the RFC 3986 parser that @@ -22,64 +19,54 @@ class FilterDescriptionLink # on every supported Ruby, and it is named here directly. PARSER = URI::RFC2396_Parser.new - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Takes the last HTTP or HTTPS URL out of the description and makes it the + # link, for feeds that carry the real destination in the body. def run - @return_feeds = [] - @pipeline.each {|feeds| - new_feeds = [] - unless feeds.nil? - feeds.items.each {|feed| - new_feeds << rewrite_link(feed) - } - end - @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) - } - @return_feeds + @pipeline.each_with_object([]) do |feeds, returned| + items = feeds.nil? ? [] : feeds.items.map { |item| rewrite(item) } + returned << Automatic::FeedMaker.create_pipeline(items) + end end private - def get_title(url) - new_title = nil - if url.class == String - url.gsub!(Regexp.new("[^#{URI::RFC2396_Parser::PATTERN::ALNUM}\/\:\?\=&~,\.\(\)#]")) {|match| ERB::Util.url_encode(match)} - begin - read_data = NKF.nkf("--utf8", URI.open(url).read) - get_text = Nokogiri::HTML.parse(read_data, nil, 'utf8').xpath('//title').text - new_title = get_text if get_text.class == String - rescue - Automatic::Log.puts("warn", "Failed in get title for: #{url}") - end - end + def rewrite(item) + link = PARSER.extract(item.description.to_s, %w[http https]).uniq.last + item.link = link unless link.nil? - new_title + item.description = '' if setting?('clear_description') + retitle(item) if setting?('get_title') + + item end - def rewrite_link(feed) - new_link = PARSER.extract(feed.description, %w{http https}).uniq.last - feed.link = new_link unless new_link.nil? + # Settings are read whatever the mapping is. This tested `@config.class == + # Hash`, which a Recipe never satisfies: the framework hands a plugin a + # Hashie::Mash, so both settings below were silently ignored in every real + # run and read only by a spec passing a plain Hash. + def setting?(name) + @config[name].to_s == '1' + end - if @config.class == Hash - if @config['clear_description'] == 1 - feed.description = "" - end + def retitle(item) + title = fetch_title(item.link) + item.title = title unless title.nil? || title.empty? + end - if @config['get_title'] == 1 - begin - new_title = get_title(feed.link) - feed.title = new_title unless new_title.nil? - rescue OpenURI::HTTPError - Automatic::Log.puts("warn", "404 Not Found in get title process.") - end - end - end + # One request per item; use FilterOne or a store plugin before this on a + # large feed. A page that cannot be read leaves the item's own title. + def fetch_title(url) + return nil unless Automatic::Http.fetchable?(url) - feed + Nokogiri::HTML.parse(Automatic::Http.read(url)).xpath('//title').text + rescue StandardError => e + Automatic::Log.puts('warn', "Failed in get title for: #{url}, #{e.message}") + nil end end end - diff --git a/plugins/filter/full_feed.rb b/plugins/filter/full_feed.rb index a3efe25..3027abc 100644 --- a/plugins/filter/full_feed.rb +++ b/plugins/filter/full_feed.rb @@ -5,77 +5,78 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Apr 29, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin - class FilterFullFeed - require 'json' Automatic.require_optional('nokogiri', needed_by: 'FilterFullFeed') - require 'open-uri' - require 'uri' + require 'json' - SITEINFO_TYPES = %w[SBM INDIVIDUAL IND SUBGENERAL SUB GENERAL GEN] + SITEINFO_TYPES = %w[SBM INDIVIDUAL IND SUBGENERAL SUB GENERAL GEN].freeze - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - @siteinfo = get_siteinfo + @siteinfo = siteinfo end + # Replaces a summary with the article body, by matching the link against a + # siteinfo database of URL patterns and XPaths and fetching the page. def run - @return_feeds = [] - @pipeline.each {|feeds| + @pipeline.each_with_object([]) do |feeds, returned| unless feeds.nil? - feeds.items.each {|feed| - feed = fulltext(feed) - } + feeds.items.each { |item| fulltext(item) } end - @return_feeds << feeds - } - @return_feeds + returned << feeds + end end private - def get_siteinfo - Automatic::Log.puts(:info, "Loading siteinfo from #{@config['siteinfo']}") - siteinfo = JSON.parse(File.read(File.join(assets_dir, @config['siteinfo']), :encoding => "UTF-8")) - siteinfo.select! { |info| SITEINFO_TYPES.include? (info['data']['type']) } - siteinfo.sort! { |a, b| - atype, btype = a['data']['type'], b['data']['type'] - SITEINFO_TYPES.index(atype) <=> SITEINFO_TYPES.index(btype) - } - return siteinfo + def siteinfo + name = @config['siteinfo'].to_s + raise ArgumentError, 'FilterFullFeed needs a siteinfo file name' if name.empty? + + Automatic::Log.puts('info', "Loading siteinfo from #{name}") + entries = JSON.parse(File.read(File.join(assets_dir, name), encoding: 'UTF-8')) + entries.select { |info| SITEINFO_TYPES.include?(info['data']['type']) } + .sort_by { |info| SITEINFO_TYPES.index(info['data']['type']) } end def assets_dir - dir = (File.expand_path('~/.automatic/assets/siteinfo')) - if File.directory?(dir) - dir - else - File.join(File.dirname(__FILE__), '..', '..', 'assets', 'siteinfo') + dir = File.expand_path('~/.automatic/assets/siteinfo') + return dir if File.directory?(dir) + + File.expand_path('../../assets/siteinfo', __dir__) + end + + def fulltext(item) + return if item.link.nil? + + info = @siteinfo.find { |entry| matches?(entry, item.link) } + if info.nil? + Automatic::Log.puts('info', "Fulltext SITEINFO not found: #{item.link}") + return end + + Automatic::Log.puts('info', "Siteinfo matched: #{info['data']['url']}") + item.description = body(item.link, info['data']['xpath']) + rescue StandardError => e + # An unreadable page leaves the item's own summary in place, which is + # what this filter is an improvement on rather than a replacement for. + Automatic::Log.puts('warn', "Failed to read fulltext for #{item.link}: #{e.message}") end - def fulltext(feed) - return feed unless feed.link - @siteinfo.each {|info| - begin - if feed.link.match(info['data']['url']) - Automatic::Log.puts(:info, "Siteinfo matched: #{info['data']['url']}") - html = Nokogiri::HTML.parse(URI.open(feed.link)) - body = html.xpath(info['data']['xpath']) - feed.description = body.to_html.encode('UTF-8', :undef => :replace) - return feed - end - rescue - return feed - end - } - Automatic::Log.puts(:info, "Fulltext SITEINFO not found: #{feed.link}") - return feed + def matches?(entry, link) + link.match?(entry['data']['url'].to_s) + rescue RegexpError + false + end + + def body(link, xpath) + document = Nokogiri::HTML.parse(Automatic::Http.read(link)) + document.xpath(xpath).to_html.encode('UTF-8', undef: :replace) end end end diff --git a/plugins/filter/github_feed.rb b/plugins/filter/github_feed.rb index f787d87..07f2626 100644 --- a/plugins/filter/github_feed.rb +++ b/plugins/filter/github_feed.rb @@ -5,34 +5,45 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 6, 2013 -# Updated:: Feb 21, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterGithubFeed - - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Converts Atom entries -- where title, id and content are elements with a + # #content -- into the flat items the rest of the pipeline expects. Needed + # because GitHub publishes Atom, not RSS. def run - @return_feeds = [] - @pipeline.each {|feeds| - new_feeds = [] - unless feeds.nil? - feeds.items.each {|feed| - Automatic::Log.puts("info", "Invoked: FilterGithubFeed") - hashie = Hashie::Mash.new - hashie.title = feed.title.content - hashie.link = feed.id.content - hashie.description = feed.content.content - new_feeds << hashie - } - end - @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) - } - @return_feeds + @pipeline.each_with_object([]) do |feeds, returned| + items = feeds.nil? ? [] : feeds.items.map { |item| flatten(item) } + returned << Automatic::FeedMaker.create_pipeline(items) + end + end + + private + + def flatten(item) + Automatic::Log.puts('info', 'Invoked: FilterGithubFeed') + entry = Hashie::Mash.new + entry.title = value(item, :title) + entry.link = value(item, :id) + entry.description = value(item, :content) + entry + end + + # An Atom element carries its text in #content; a field that is already a + # string is used as it stands, so that a pipeline which has been through + # another filter first is not a NoMethodError. + def value(item, name) + return nil unless item.respond_to?(name) + + field = item.send(name) + field.respond_to?(:content) ? field.content : field end end end diff --git a/plugins/filter/google_news.rb b/plugins/filter/google_news.rb deleted file mode 100644 index 3953d45..0000000 --- a/plugins/filter/google_news.rb +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Filter::GoogleNews -# Author: id774 (More info: http://id774.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Oct 12, 2014 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class FilterGoogleNews - require 'uri' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - end - - def run - @return_feeds = [] - @pipeline.each {|feeds| - new_feeds = [] - unless feeds.nil? - feeds.items.each {|feed| - new_feeds << rewrite_link(feed) unless feed.link.nil? - } - end - @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) - } - @return_feeds - end - - private - - def rewrite_link(feed) - if feed.link.class == String - if feed.link.index("http://news.google.com") - matched = feed.link.match(/(&url=)/) - unless matched.nil? - new_link = matched.post_match - feed.link = new_link unless new_link.nil? - end - end - end - - feed - end - end -end - diff --git a/plugins/filter/ignore.rb b/plugins/filter/ignore.rb index ba6c206..483b9d2 100644 --- a/plugins/filter/ignore.rb +++ b/plugins/filter/ignore.rb @@ -5,61 +5,48 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 22, 2012 -# Updated:: Feb 21, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterIgnore - def initialize(config, pipeline=[]) - @config = config + FIELDS = %i[title link description].freeze + + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Drops items containing any listed keyword. Matching is a substring test, + # so an empty keyword drops everything. def run - @return_feeds = [] - @pipeline.each {|feeds| - new_feeds = [] - unless feeds.nil? - feeds.items.each {|items| - new_feeds << items if exclude(items) == false - } - end - @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0 - } - @return_feeds + @pipeline.each_with_object([]) do |feeds, returned| + kept = feeds.nil? ? [] : feeds.items.reject { |item| exclude?(item) } + returned << Automatic::FeedMaker.create_pipeline(kept) unless kept.empty? + end end private - def detect_exclude(item, evaluation, reason) - begin - if item.include?(evaluation) - Automatic::Log.puts("info", "Excluded by #{reason}: #{item}") - return true + + def exclude?(item) + FIELDS.any? do |field| + Array(@config[field.to_s]).any? do |keyword| + excluded?(item.send(field), keyword.to_s.chomp, field) end - rescue NoMethodError - Automatic::Log.puts("warn", "Invalid feed detected in ignore process with #{item}") - return false end end - def exclude(items) - detection = false - unless @config['title'].nil? - @config['title'].each {|e| - detection = true if detect_exclude(items.title, e.chomp, 'title') - } - end - unless @config['link'].nil? - @config['link'].each {|e| - detection = true if detect_exclude(items.link, e.chomp, 'link') - } - end - unless @config['description'].nil? - @config['description'].each {|e| - detection = true if detect_exclude(items.description, e.chomp, 'description') - } + # An item whose field is missing is kept, with a warning. + def excluded?(value, keyword, field) + unless value.respond_to?(:include?) + Automatic::Log.puts('warn', "Invalid feed detected in ignore process with #{value}") + return false end - detection + + return false unless value.include?(keyword) + + Automatic::Log.puts('info', "Excluded by #{field}: #{value}") + true end end end diff --git a/plugins/filter/image.rb b/plugins/filter/image.rb index 6c87e0b..0b4be39 100644 --- a/plugins/filter/image.rb +++ b/plugins/filter/image.rb @@ -5,40 +5,49 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Sep 18, 2012 -# Updated:: Apr 5, 2013 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterImage + require 'uri' - def initialize(config, pipeline=[]) - @config = config + # webp and avif are here because they are what an image link on the + # current web frequently is; tif joins tiff for the same reason. + EXTENSIONS = /\.(jpe?g|gif|png|tiff?|webp|avif)\z/i + + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Sets link to nil unless it names an image. Note that the items are kept: + # their links are blanked, and the plugins after this one skip an item + # whose link is nil. def run - @return_feeds = [] - @pipeline.each {|feeds| - return_feed_items = [] - unless feeds.nil? - feeds.items.each {|feed| - feed.link = image?(feed.link) unless feed.link.nil? - } - @return_feeds << feeds + @pipeline.each_with_object([]) do |feeds, returned| + next if feeds.nil? + + feeds.items.each do |item| + item.link = nil unless item.link.nil? || image?(item.link) end - } - @return_feeds + returned << feeds + end end private + + # The test is on the path, so that a link carrying a query string -- which + # is how an image is served by most of what serves images now -- is still + # recognised. A string that will not parse is tested whole, as before. def image?(link) - case link - when /\.jpe?g\Z/i then link - when /\.gif\Z/i then link - when /\.png\Z/i then link - when /\.tiff\Z/i then link - else nil - end + EXTENSIONS.match?(path(link)) + end + + def path(link) + URI.parse(link).path.to_s + rescue URI::InvalidURIError + link end end end diff --git a/plugins/filter/image_source.rb b/plugins/filter/image_source.rb index e766c27..bffe7ed 100644 --- a/plugins/filter/image_source.rb +++ b/plugins/filter/image_source.rb @@ -5,71 +5,73 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 28, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterImageSource - require 'net/http' Automatic.require_optional('nokogiri', needed_by: 'FilterImageSource') - require 'open-uri' require 'uri' - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Replaces each item with one item per image found: the images in the + # description, or, where it has none, the images on the page the link + # points at. The second case reaches the network. def run - @return_feeds = [] - @pipeline.each {|feeds| - new_feeds = Array.new - unless feeds.nil? - feeds.items.each {|feed| - arr = rewrite_link(feed) - if arr.length > 0 - arr.each {|link| - Automatic::Log.puts("info", "Extract Image: #{link}") - hashie = Hashie::Mash.new - hashie.title = 'FilterImageSource' - hashie.link = link - new_feeds << hashie - } - end - } - end - @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) - } - @return_feeds + @pipeline.each_with_object([]) do |feeds, returned| + items = feeds.nil? ? [] : feeds.items.flat_map { |item| extract(item) } + returned << Automatic::FeedMaker.create_pipeline(items) + end end private - def rewrite_link(feed) - array = Array.new - feed.description.scan(/ e + Automatic::Log.puts('warn', "Failed to read images from #{link}: #{e.message}") + [] + end + + # The images of an HTML fragment, as absolute URLs. This was a scan for + # ` e + Automatic::Log.puts('warn', "Undefined field detected in feed: #{e.message}") end end end diff --git a/plugins/filter/sort.rb b/plugins/filter/sort.rb index 2653c05..d639a20 100644 --- a/plugins/filter/sort.rb +++ b/plugins/filter/sort.rb @@ -5,35 +5,26 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Mar 23, 2012 -# Updated:: Jan 23, 2013 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterSort - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline + def initialize(config, pipeline = []) + @config = config || {} + @pipeline = pipeline + @ascending = @config['sort'].to_s == 'asc' end + # Sorts each feed's items by date. Items must carry one; a feed built from + # a source without dates fails here. def run - @return_feeds = [] - @pipeline.each { |feeds| - return_feed_items = [] - unless feeds.nil? - if @config['sort'] == "asc" - feeds.items.sort!{|a,b| - a.date <=> b.date - } - else - feeds.items.sort!{|a,b| - - (a.date <=> b.date) - } - end - @return_feeds << feeds - end - } - @return_feeds + @pipeline.each_with_object([]) do |feeds, returned| + next if feeds.nil? + + feeds.items.sort! { |a, b| @ascending ? a.date <=> b.date : b.date <=> a.date } + returned << feeds + end end end end diff --git a/plugins/filter/tumblr_resize.rb b/plugins/filter/tumblr_resize.rb index 5d9cff0..cdb7827 100644 --- a/plugins/filter/tumblr_resize.rb +++ b/plugins/filter/tumblr_resize.rb @@ -5,38 +5,44 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 28, 2012 -# Updated:: Apr 5, 2013 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class FilterTumblrResize + # Tumblr has served images under two URL schemes. The older one carries + # the size as a suffix on the file name -- tumblr_xxx_500.jpg -- and is + # what images uploaded before 2019 still use. The newer one carries it as + # a path segment -- /s540x810/ -- and is what everything since uses. + # Both are rewritten to the largest variant the scheme offers. + LEGACY_SIZE = /_(?:75sq|100|250|400|500)(?=\.[a-zA-Z0-9]+\z)/ + LEGACY_LARGEST = '_1280' - def initialize(config, pipeline=[]) - @config = config + PATH_SIZE = %r{/s\d+x\d+/} + PATH_LARGEST = '/s1280x1920/' + + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Assumes FilterImage or FilterImageSource has already put an image URL in + # the link. def run - @return_feeds = [] - @pipeline.each {|feeds| - img_url = "" + @pipeline.each_with_object([]) do |feeds, returned| unless feeds.nil? - feeds.items.each {|feed| - feed.link = resize(feed.link) unless feed.link.nil? - } + feeds.items.each do |item| + item.link = resize(item.link) unless item.link.nil? + end end - @return_feeds << feeds - } - @return_feeds + returned << feeds + end end private - def resize(string) - string = string.gsub("_75sq\.", "_1280\.") - string = string.gsub("_100\.", "_1280\.") - string = string.gsub("_250\.", "_1280\.") - string = string.gsub("_400\.", "_1280\.") - string = string.gsub("_500\.", "_1280\.") + + def resize(link) + link.sub(LEGACY_SIZE, LEGACY_LARGEST).sub(PATH_SIZE, PATH_LARGEST) end end end diff --git a/plugins/notify/ikachan.rb b/plugins/notify/ikachan.rb index 4223061..69b314f 100644 --- a/plugins/notify/ikachan.rb +++ b/plugins/notify/ikachan.rb @@ -5,92 +5,121 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Mar 7, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. # # Description:: Posts each item to an IRC channel through an ikachan # HTTP-to-IRC gateway, which the operator runs themselves. -require 'active_support/core_ext/object/blank' +require 'net/http' +require 'uri' module Automatic::Plugin class Ikachan - require 'rubygems' - require 'time' - require 'net/http' - require 'uri' + OPEN_TIMEOUT = 10 + READ_TIMEOUT = 30 attr_accessor :params def initialize - @params = { - "url" => "", - "port" => "", - "channels" => [], - "command" => "", - } + @params = { 'url' => '', 'port' => '', 'channels' => [], 'command' => '' } end - def post(link, title = "") - message = build_message(link, title) - uri = URI.parse(endpoint_url) - proxy_class = Net::HTTP::Proxy(ENV["PROXY"], 8080) - http = proxy_class.new(uri.host, uri.port) - http.start do |http| - @params['channels'].split(",").each do|channel| - # send join command to make sure when if ikachan is not in the channel - http.post("/join", "channel=#{channel}") - res = http.post(uri.path, %Q(channel=#{channel}&message=#{message})) - if res.code == "200" - Automatic::Log.puts(:info, "Success: #{message}") - else - Automatic::Log.puts(:error, "#{res.code} Error: #{message}") - end + def post(link, title = '') + message = build_message(link, title) + uri = endpoint + + start(uri) do |http| + channels.each do |channel| + # Join first, in case the gateway is not in the channel. + http.request(form('/join', channel: channel)) + response = http.request(form(uri.path, channel: channel, message: message)) + log(response, message) end end end private - def endpoint_url - "#{@params['url']}:#{@params['port']}/#{@params['command']}" + # A gateway reached over https is spoken to over https. The earlier + # version built a plain connection whatever the URL said. + def start(uri, &block) + proxy = Net::HTTP.Proxy(ENV.fetch('PROXY', nil), 8080) + proxy.start(uri.host, uri.port, + use_ssl: uri.scheme == 'https', + open_timeout: OPEN_TIMEOUT, + read_timeout: READ_TIMEOUT, &block) + end + + # Form-encoded by Net::HTTP rather than interpolated into the body, so + # that a title carrying an ampersand or a space reaches the channel as it + # was written instead of splitting the request. + def form(path, params) + request = Net::HTTP::Post.new(path) + request.set_form_data(params) + request + end + + def channels + value = @params['channels'] + value.is_a?(Array) ? value : value.to_s.split(',') + end + + def endpoint + URI.parse("#{@params['url']}:#{@params['port']}/#{@params['command']}") + end + + def log(response, message) + if response.code == '200' + Automatic::Log.puts(:info, "Success: #{message}") + else + Automatic::Log.puts(:error, "#{response.code} Error: #{message}") + end end def build_message(link, title) - message = "" - message += "#{title.to_s} - " unless title.blank? - message += link.to_s - message + message = '' + message += "#{title} - " unless title.nil? || title.to_s.empty? + message + link.to_s end end class NotifyIkachan attr_accessor :ikachan - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline @ikachan = Ikachan.new @ikachan.params = { - "url" => @config['url'], - "port" => @config['port'] || "4979", - "channels" => @config['channels'].split(',').map{ |v| v.match(/^#/) ? v : "#" + v }, # Prifixing '#' - "command" => @config['command'] || "notice", + 'url' => @config['url'], + 'port' => @config['port'] || '4979', + 'channels' => channels, + 'command' => @config['command'] || 'notice' } end + # Returns the pipeline unchanged: a Notify plugin sends a notification and + # nothing else. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - Automatic::Log.puts("info", %Q(Ikachan: [#{feed.link}] sending with params #{ikachan.params.to_s}...)) - ikachan.post(feed.link, feed.title) - sleep ||= @config['interval'].to_i - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each do |feed| + Automatic::Log.puts('info', "Ikachan: [#{feed.link}] sending to #{channels.join(',')}...") + ikachan.post(feed.link, feed.title) + sleep(@config['interval'].to_i) end - } + end @pipeline end + + private + + # Comma separated, with a leading '#' added where it is absent. + def channels + @config['channels'].to_s.split(',').map { |name| name.start_with?('#') ? name : "##{name}" } + end end end diff --git a/plugins/provide/fluentd.rb b/plugins/provide/fluentd.rb index fc2ff67..680b4e7 100644 --- a/plugins/provide/fluentd.rb +++ b/plugins/provide/fluentd.rb @@ -5,35 +5,52 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jul 12, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class ProvideFluentd Automatic.require_optional('fluent-logger', needed_by: 'ProvideFluentd') - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - @mode = @config['mode'] - @fluentd = Fluent::Logger::FluentLogger.open(nil, - host = @config['host'], - port = @config['port']) unless @mode == 'test' + @mode = @config['mode'] + @fluentd = open_logger unless test? end + # Posts each item's content_encoded to Fluentd, which must be something + # Fluentd accepts as a record. Distinct from PublishFluentd, which posts + # the item's fields. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - begin - @fluentd.post(@config['tag'], feed.content_encoded) unless @mode == 'test' - rescue - Automatic::Log.puts("error", "Fluent::Logger.post failed, the content_encoded of item may be not kind of Hash.") - end - } - end - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each { |feed| post(feed) } + end @pipeline end + + private + + def open_logger + Fluent::Logger::FluentLogger.open(nil, + host: @config['host'], + port: @config['port'].to_i) + end + + def post(feed) + return if test? + + @fluentd.post(@config['tag'], feed.content_encoded) + rescue StandardError => e + Automatic::Log.puts('error', + 'Fluent::Logger.post failed, the content_encoded of item may ' \ + "be not kind of Hash: #{e.message}") + end + + def test? + @mode == 'test' + end end end diff --git a/plugins/publish/amazon_s3.rb b/plugins/publish/amazon_s3.rb index 0e3712b..616948f 100644 --- a/plugins/publish/amazon_s3.rb +++ b/plugins/publish/amazon_s3.rb @@ -5,56 +5,87 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 24, 2014 -# Updated:: Feb 24, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishAmazonS3 require 'uri' - require 'aws-sdk' - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - s3 = AWS::S3.new( - :access_key_id => @config['access_key'], - :secret_access_key => @config['secret_key'] - ) - @bucket = s3.buckets[@config['bucket_name']] - @mode = @config['mode'] + @mode = @config['mode'] end + # Uploads the files whose link is a file URI, normally after StoreFile. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - unless feed.link.nil? - begin - uri = URI.parse(feed.link) - if uri.scheme == 'file' - upload_amazons3(uri.path, @config['target_path']) - else - Automatic::Log.puts("warn", "Skip feed due to uri scheme is not file.") - end - rescue - Automatic::Log.puts("error", "Error detected with #{feed.link} in uploading AmazonS3.") - end - end - } - end - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each { |feed| publish(feed) } + end @pipeline end private - def upload_amazons3(filename, target_path) - target = File.join(target_path, File.basename(filename)) - object = @bucket.objects[target] - source = Pathname.new(filename) - object.write(source) unless @mode == "test" - Automatic::Log.puts("info", "Uploaded: file #{source} to the bucket #{target} on #{@bucket.name}.") - return source, target + def publish(feed) + return if feed.link.nil? + + uri = URI.parse(feed.link) + if uri.scheme == 'file' + upload(uri.path) + else + Automatic::Log.puts('warn', 'Skip feed due to uri scheme is not file.') + end + rescue StandardError => e + Automatic::Log.puts('error', + "Error detected with #{feed.link} in uploading AmazonS3: #{e.message}") + end + + def upload(path) + key = target_key(path) + File.open(path, 'rb') { |body| s3.put_object(bucket: bucket, key: key, body: body) } unless test? + Automatic::Log.puts('info', "Uploaded: file #{path} to the key #{key} on #{bucket}.") + [path, key] + end + + def target_key(path) + File.join(@config['target_path'].to_s, File.basename(path)).sub(%r{\A/}, '') + end + + def bucket + @config['bucket_name'].to_s + end + + def test? + @mode == 'test' + end + + # AWS SDK for Ruby v3, which is the SDK AWS publishes and maintains. The + # gem is required here rather than at the top of the file, so that a + # Recipe running this plugin in `mode: test` -- and this plugin's own + # specs -- need neither the gem nor an account. + # + # Credentials are the Recipe's where it carries them, and the SDK's + # default chain -- environment, shared profile, instance role -- where it + # does not, which is the way to run this without a secret in a file. + def s3 + @s3 ||= begin + Automatic.require_optional('aws-sdk-s3', needed_by: 'PublishAmazonS3') + Aws::S3::Client.new(**client_options) + end + end + + def client_options + options = {} + options[:region] = @config['region'].to_s unless @config['region'].nil? + unless @config['access_key'].nil? + options[:access_key_id] = @config['access_key'].to_s + options[:secret_access_key] = @config['secret_key'].to_s + end + options end end end diff --git a/plugins/publish/console.rb b/plugins/publish/console.rb index 5506699..d23f9d9 100644 --- a/plugins/publish/console.rb +++ b/plugins/publish/console.rb @@ -5,27 +5,28 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 23, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishConsole require 'pp' - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - @output = $stdout + # Held in an instance variable rather than written to $stdout directly, + # so that a spec can substitute it. See doc/PLUGINS.md section 3.6. + @output = $stdout end + # Prints each item, the plugin to end a Recipe with while writing it. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - @output.puts(feed.pretty_inspect) - } - end - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each { |feed| @output.puts(feed.pretty_inspect) } + end @pipeline end end diff --git a/plugins/publish/console_link.rb b/plugins/publish/console_link.rb index df0023b..b421594 100644 --- a/plugins/publish/console_link.rb +++ b/plugins/publish/console_link.rb @@ -5,27 +5,28 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 02, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishConsoleLink - require 'pp' - - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - @output = $stdout + @output = $stdout end + # Prints each item's link, one per line, and nothing else. Useful in a + # pipe, which is why an item whose link a filter has blanked prints + # nothing rather than an empty line. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - @output.puts(feed.link) - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each do |feed| + @output.puts(feed.link) unless feed.link.nil? end - } + end @pipeline end end diff --git a/plugins/publish/eject.rb b/plugins/publish/eject.rb index ca1fa01..feb814a 100644 --- a/plugins/publish/eject.rb +++ b/plugins/publish/eject.rb @@ -5,39 +5,60 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 9, 2013 -# Updated:: May 16, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishEject + # The command that opens the tray and the command that closes it again, + # per platform, as argument vectors. Nothing here goes through a shell. + COMMANDS = { + 'eject' => [%w[eject], %w[eject -t]], # GNU/Linux + 'drutil' => [%w[drutil tray eject], %w[drutil tray close]] # macOS + }.freeze - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Opens and closes the optical drive once per item. A physical + # notification, and the plugin that needs a machine with a tray. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - unless feed.link.nil? - `#{eject_cmd}` - Automatic::Log.puts('info', "Eject: #{feed.link}") - - interval = @config['interval'].to_i unless @config['interval'].nil? unless @config.nil? - sleep ||= @config['interval'].to_i - end - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each do |feed| + next if feed.link.nil? + + eject + Automatic::Log.puts('info', "Eject: #{feed.link}") + sleep(@config['interval'].to_i) end - } + end @pipeline end - def eject_cmd - if `which eject` != '' # linux - 'eject ; eject -t' - elsif `which drutil` != '' # mac - 'drutil tray eject ; drutil tray close' + private + + def eject + commands = COMMANDS[command_name] + if commands.nil? + Automatic::Log.puts('warn', 'No eject command found on this system.') + return + end + + commands.each { |command| system(*command) } + end + + # Looked up on PATH rather than by running `which` in a shell. + def command_name + @command_name ||= COMMANDS.keys.find { |name| executable?(name) } + end + + def executable?(name) + ENV['PATH'].to_s.split(File::PATH_SEPARATOR).any? do |dir| + File.executable?(File.join(dir, name)) end end end diff --git a/plugins/publish/fluentd.rb b/plugins/publish/fluentd.rb index 2c23e5f..c94dd60 100644 --- a/plugins/publish/fluentd.rb +++ b/plugins/publish/fluentd.rb @@ -5,41 +5,59 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 21, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishFluentd Automatic.require_optional('fluent-logger', needed_by: 'PublishFluentd') - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - @mode = @config['mode'] - @fluentd = Fluent::Logger::FluentLogger.open(nil, - host = @config['host'], - port = @config['port']) unless @mode == 'test' + @mode = @config['mode'] + @fluentd = open_logger unless test? end + # Posts each item's title, link, description, content_encoded and a + # timestamp to Fluentd. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - begin - @fluentd.post(@config['tag'], { - :title => feed.title, - :link => feed.link, - :description => feed.description, - :content => feed.content_encoded, - :created_at => Time.now.strftime("%Y/%m/%d %X") - }) unless @mode == 'test' - rescue - Automatic::Log.puts("warn", "Skip feed due to fault in forward.") - end - } - end - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each { |feed| post(feed) } + end @pipeline end + + private + + def open_logger + Fluent::Logger::FluentLogger.open(nil, + host: @config['host'], + port: @config['port'].to_i) + end + + def post(feed) + return if test? + + @fluentd.post(@config['tag'], record(feed)) + rescue StandardError => e + Automatic::Log.puts('warn', "Skip feed due to fault in forward: #{e.message}") + end + + def record(feed) + { + title: feed.title, + link: feed.link, + description: feed.description, + content: feed.content_encoded, + created_at: Time.now.strftime('%Y/%m/%d %X') + } + end + + def test? + @mode == 'test' + end end end diff --git a/plugins/publish/google_calendar.rb b/plugins/publish/google_calendar.rb deleted file mode 100644 index 28ec7e7..0000000 --- a/plugins/publish/google_calendar.rb +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Googlecalendar -# Author: id774 (More info: http://id774.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Feb 24, 2012 -# Updated:: Jan 15, 2014 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class Googlecalendar - attr_accessor :user, :feed - - def initialize - @user = { - "username" => "", - "password" => "" - } - @feed = 'http://www.google.com/calendar/feeds/default/private/full' - end - - def add(arg) - date = nil - time_st = nil - time_en = nil - text = '' - - # Parse Date - require 'date' - if /^([0-9]+\/[0-9]+\/[0-9]+)\s*/ =~ arg - # yyyy/mm/dd - datestr = $1 - text = $' - begin - date = Date.parse(datestr) - rescue ArgumentError - raise "不正な日付形式-1： [#{datestr}]" - end - end - - Automatic::Log.puts("info", "Date ： #{date}") - Automatic::Log.puts("info", "Title ： #{text}") - - # Register to calendar - require 'rubygems' - require 'gcalapi' - - cal = GoogleCalendar::Calendar.new(GoogleCalendar::Service.new( - @user["username"], @user["password"]), @feed) - event = cal.create_event - event.title = text - event.st = Time.mktime(date.year, date.month, date.day) - event.en = event.st - event.allday = true - event.save! - end - end - - class PublishGoogleCalendar - attr_accessor :hb - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - - @gc = Googlecalendar.new - @gc.user = { - "hatena_id" => @config['username'], - "password" => @config['password'] - } - end - - def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - @gc.add('今日 ' + feed.title) - sleep ||= @config['interval'].to_i - } - end - } - @pipeline - end - end -end diff --git a/plugins/publish/hatena_bookmark.rb b/plugins/publish/hatena_bookmark.rb index 460407a..38fee5d 100644 --- a/plugins/publish/hatena_bookmark.rb +++ b/plugins/publish/hatena_bookmark.rb @@ -5,108 +5,123 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 22, 2012 -# Updated:: Jan 15, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. +# +# STATUS: Needs rework. This speaks the WSSE AtomPub interface, which Hatena +# has superseded with an OAuth one. The service and the bookmarking API are +# both current; this client is not, and restoring it means the current +# endpoint and the current authentication rather than a change of a few lines. +# See doc/PLUGINS.md section 6.7. +# +# The transport was corrected in the meantime: the request is made over HTTPS, +# so that an operator who runs this does not put a password digest on the wire +# in the clear. That is a defect worth not shipping whatever the plugin's +# status is; it is not a claim that the plugin works. + +require 'digest/sha1' +require 'net/http' +require 'securerandom' +require 'time' +require 'uri' module Automatic::Plugin class HatenaBookmark - require 'rubygems' - require 'time' - require 'digest/sha1' - require 'net/http' - require 'uri' - #require 'nkf' + ENDPOINT = 'https://b.hatena.ne.jp/atom/post' + OPEN_TIMEOUT = 10 + READ_TIMEOUT = 30 attr_accessor :user def initialize - @user = { - "hatena_id" => "", - "password" => "" - } + @user = { 'hatena_id' => '', 'password' => '' } end def wsse(hatena_id, password) - # Unique value - nonce = [Time.now.to_i.to_s].pack('m').gsub(/\n/, '') - now = Time.now.utc.iso8601 + nonce = SecureRandom.random_bytes(16) + now = Time.now.utc.iso8601 + digest = [Digest::SHA1.digest(nonce + now + password.to_s)].pack('m0') - # Base64 encoding for SHA1 Digested strings - digest = [Digest::SHA1.digest(nonce + now + password)].pack("m").gsub(/\n/, '') + { 'X-WSSE' => format('UsernameToken Username="%s", PasswordDigest="%s", ' \ + 'Nonce="%s", Created="%s"', + hatena_id, digest, [nonce].pack('m0'), now) } + end - {'X-WSSE' => sprintf( - %Q, - hatena_id, digest, nonce, now) - } + def to_xml(link, summary) + <<~XML + + dummy + + #{summary} + + XML end - def toXml(link, summary) - %Q( - - dummy - - #{summary} - - ) + def post(url, comment) + Automatic::Log.puts('info', "Bookmarking: #{url}") + uri = URI.parse(ENDPOINT) + request = Net::HTTP::Post.new(uri.path, wsse(@user['hatena_id'], @user['password'])) + request.body = to_xml(url, comment) + + response = start(uri) { |http| http.request(request) } + log(response, url, comment) end - def post(b_url, b_comment) - Automatic::Log.puts("info", "Bookmarking: #{b_url}") - url = "http://b.hatena.ne.jp/atom/post" - header = wsse(@user["hatena_id"], @user["password"]) - uri = URI.parse(url) - proxy_class = Net::HTTP::Proxy(ENV["PROXY"], 8080) - http = proxy_class.new(uri.host) - http.start { |http| - # b_url = NKF.nkf('-w', b_url) - # b_comment = NKF.nkf('-w', b_comment) - res = http.post(uri.path, toXml(b_url, b_comment), header) - if res.code == "201" then - message = "Success: #{b_url}" - message += " Comment: #{b_comment}" unless b_comment.nil? - Automatic::Log.puts(:info, message) - else - Automatic::Log.puts(:error, "#{res.code} Error: #{b_url}") - end - } + private + + def start(uri, &block) + proxy = Net::HTTP.Proxy(ENV.fetch('PROXY', nil), 8080) + proxy.start(uri.host, uri.port, + use_ssl: true, + open_timeout: OPEN_TIMEOUT, + read_timeout: READ_TIMEOUT, &block) + end + + def log(response, url, comment) + if response.code == '201' + message = "Success: #{url}" + message += " Comment: #{comment}" unless comment.nil? + Automatic::Log.puts(:info, message) + else + Automatic::Log.puts(:error, "#{response.code} Error: #{url}") + end end end class PublishHatenaBookmark attr_accessor :hb - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline @hb = HatenaBookmark.new @hb.user = { - "hatena_id" => @config['username'], - "password" => @config['password'] + 'hatena_id' => @config['username'], + 'password' => @config['password'] } end def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - hb.post(rewrite(feed.link), nil) - sleep ||= @config['interval'].to_i - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each do |feed| + hb.post(absolute(feed.link), nil) + sleep(@config['interval'].to_i) end - } + end @pipeline end private - def rewrite(string) - if /^https?:\/\/.*$/ =~ string - return string - elsif /^\/\/.*$/ =~ string - return "http:" + string - else - return "http://" + string - end + + def absolute(link) + string = link.to_s + return string if string.match?(%r{\Ahttps?://}) + return "https:#{string}" if string.start_with?('//') + + "https://#{string}" end end end diff --git a/plugins/publish/hipchat.rb b/plugins/publish/hipchat.rb deleted file mode 100644 index 168d2a0..0000000 --- a/plugins/publish/hipchat.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Hipchat -# Author: Kohei Hasegawa (More info: http://github.com/banyan) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Jun 6, 2013 -# Updated:: Jan 15, 2014 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - require 'hipchat' - - class PublishHipchat - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - @options = { - 'color' => 'yellow', - 'notify' => false - } - @options.merge!(@config.select { |k, v| @options.include?(k) }) - @client = HipChat::Client.new(@config['api_token'])[@config['room_id']] - end - - def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - @client.send(@config['username'], feed.description, @options) - Automatic::Log.puts("info", "Hipchat post: #{feed.description.gsub(/[\r\n]/,'')[0..50]}...") rescue nil - rescue => e - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, #{e.message}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - sleep ||= @config['interval'].to_i - } - end - } - @pipeline - end - end -end diff --git a/plugins/publish/instapaper.rb b/plugins/publish/instapaper.rb index b663a04..8218040 100644 --- a/plugins/publish/instapaper.rb +++ b/plugins/publish/instapaper.rb @@ -5,85 +5,94 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 9, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin + # The Instapaper Simple API: HTTPS, basic authentication, form parameters. + # @see https://www.instapaper.com/api/simple class Instapaper - # @see http://www.instapaper.com/api/simple - require "net/http" - require "openssl" + require 'net/http' + require 'openssl' + require 'uri' + + HOST = 'www.instapaper.com' + OPEN_TIMEOUT = 10 + READ_TIMEOUT = 30 + + class Error < StandardError; end def initialize(username, password = '') @username = username - @password = password + @password = password.to_s request(:authenticate) end def add(url, title = '', selection = '') - params = { - :url => url, - :title => title, - :selection => selection - } - res = request(:add, params) - if res.code == "201" then - message = "Success: #{url}" - message += " Title: #{title}" unless title.nil? - Automatic::Log.puts(:info, message) - else - Automatic::Log.puts(:error, "#{res.code} Error: #{url}") - raise - end + response = request(:add, url: url, title: title, selection: selection) + raise Error, "Instapaper answered #{response.code} for #{url}" unless response.code == '201' + + message = "Success: #{url}" + message += " Title: #{title}" unless title.nil? + Automatic::Log.puts(:info, message) + response end private + # TLS with the certificate verified, which is Net::HTTP's own default and + # is stated here because this plugin used to turn it off. def request(method, params = {}) - request = Net::HTTP::Post.new('/api/' + method.to_s) - request.basic_auth(@username, @password) - request.set_form_data(params) - http = Net::HTTP.new('www.instapaper.com', 443) - http.use_ssl = true - http.verify_mode = OpenSSL::SSL::VERIFY_PEER - http.start { http.request(request) } + post = Net::HTTP::Post.new("/api/#{method}") + post.basic_auth(@username, @password) + post.set_form_data(params) + + Net::HTTP.start(HOST, 443, + use_ssl: true, + verify_mode: OpenSSL::SSL::VERIFY_PEER, + open_timeout: OPEN_TIMEOUT, + read_timeout: READ_TIMEOUT) { |http| http.request(post) } end end class PublishInstapaper - attr_accessor :instapaper - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - - @instapaper = Instapaper.new( - @config['email'], - @config['password'] - ) + def initialize(config, pipeline = []) + @config = config || {} + @pipeline = pipeline + @instapaper = Instapaper.new(@config['email'], @config['password']) end + # Adds each item to Instapaper. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - Automatic::Log.puts("info", "add: #{feed.link}") - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - instapaper.add(feed.link, feed.title, feed.description) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in publish to instapaper.") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - sleep ||= @config['interval'].to_i - } - end - } + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each { |feed| add(feed) } + end @pipeline end + + private + + def add(feed) + Automatic::Log.puts('info', "add: #{feed.link}") + retries = 0 + retry_max = @config['retry'].to_i + begin + instapaper.add(feed.link, feed.title, feed.description) + rescue StandardError => e + retries += 1 + # The message names the item, never the account or the password. + Automatic::Log.puts('error', + "ErrorCount: #{retries}, Fault in publish to instapaper: #{e.message}") + if retries <= retry_max + sleep(@config['interval'].to_i) + retry + end + end + sleep(@config['interval'].to_i) + end end end diff --git a/plugins/publish/memcached.rb b/plugins/publish/memcached.rb index ec005db..67cbee4 100644 --- a/plugins/publish/memcached.rb +++ b/plugins/publish/memcached.rb @@ -5,42 +5,46 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 25, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishMemcached Automatic.require_optional('dalli', needed_by: 'PublishMemcached') - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - @cache = Dalli::Client.new( - @config['host'] + ":" + - @config['port']) + # Interpolated rather than concatenated: `port: 11211` in a Recipe is an + # Integer, and String#+ ended the run on it. + @cache = Dalli::Client.new("#{@config['host']}:#{@config['port']}") end + # Collects the whole pipeline into one hash keyed by link and stores it + # under a single key, replacing the previous value. def run - hash = {} - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - hash[feed.link] = - { - :title => feed.title, - :description => feed.description, - :content => feed.content_encoded, - :created_at => Time.now.strftime("%Y/%m/%d %X") - } + @cache.set(@config['key'], collect) + @pipeline + rescue StandardError => e + Automatic::Log.puts('warn', "Skip feed due to fault in put to memcached: #{e.message}") + @pipeline + end + + private + + def collect + @pipeline.each_with_object({}) do |feeds, hash| + next if feeds.nil? + + feeds.items.each do |feed| + hash[feed.link] = { + title: feed.title, + description: feed.description, + content: feed.content_encoded, + created_at: Time.now.strftime('%Y/%m/%d %X') } end - } - begin - @cache.set(@config['key'], hash) - rescue - Automatic::Log.puts("warn", "Skip feed due to fault in put to memcached.") end - @pipeline end end end diff --git a/plugins/publish/pocket.rb b/plugins/publish/pocket.rb deleted file mode 100644 index f87c434..0000000 --- a/plugins/publish/pocket.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Pocket -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 15, 2013 -# Updated:: Jan 15, 2014 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class PublishPocket - require 'pocket' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - Pocket.configure do |c| - c.consumer_key = @config['consumer_key'] - c.access_token = @config['access_token'] - end - @client = Pocket.client - end - - def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - @client.add(:url => feed.link) - Automatic::Log.puts("info", "add: #{feed.link}") - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in publish to pocket.") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - sleep ||= @config['interval'].to_i - } - end - } - @pipeline - end - end -end diff --git a/plugins/publish/twitter.rb b/plugins/publish/twitter.rb deleted file mode 100644 index 7f6e738..0000000 --- a/plugins/publish/twitter.rb +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Twitter -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 5, 2013 -# Updated:: May 5, 2013 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class PublishTwitter - require 'twitter' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - - Twitter.configure do |conf| - conf.consumer_key = @config['consumer_key'] - conf.consumer_secret = @config['consumer_secret'] - conf.oauth_token = @config['oauth_token'] - conf.oauth_token_secret = @config['oauth_token_secret'] - end - @twitter = Twitter - - if @config['tweet_tmp'] == nil - @tweet_tmp = '{title} {link}' - else - @tweet_tmp = @config['tweet_tmp'] - end - - end - - def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - Automatic::Log.puts("info", "Publish Tweet: #{feed.link}") - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - tweet = @tweet_tmp.gsub(/\{(.+?)\}/) do |text| - feed.__send__($1) - end - @twitter.update(tweet) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in publish to twitter.") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - sleep ||= @config['interval'].to_i - } - end - } - @pipeline - end - end -end diff --git a/plugins/store/database.rb b/plugins/store/database.rb index ea4578a..8157171 100644 --- a/plugins/store/database.rb +++ b/plugins/store/database.rb @@ -5,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 27, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. # # The SQLite storage the store plugins share. ActiveRecord and sqlite3 are the @@ -20,58 +20,52 @@ module Automatic::Plugin module Database + # Yields each item that is not already in the table, and returns a pipeline + # of those items. An item with no link is neither stored nor passed on. def for_each_new_feed prepare_database - existing_records = model_class.all - @return_feeds = [] - @pipeline.each {|feeds| - unless feeds.nil? - new_feeds = [] - feeds.items.each {|feed| - unless feed.link.nil? - if unique_keys.length > 1 - detection = existing_records.detect {|b| b.try(unique_keys[0]) == feed.link || b.try(unique_keys[1]) == feed.title } - else - detection = existing_records.detect {|b| b.try(unique_keys[0]) == feed.link } - end - unless detection - yield(feed) - new_feeds << feed - end - end - } - @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0 - end - } - @return_feeds + + @pipeline.each_with_object([]) do |feeds, returned| + next if feeds.nil? + + new_feeds = feeds.items.reject { |feed| feed.link.nil? || stored?(feed) } + new_feeds.each { |feed| yield(feed) } + returned << Automatic::FeedMaker.create_pipeline(new_feeds) unless new_feeds.empty? + end end private + # Asked of the database rather than of every row loaded into memory, which + # is what this did before: a store whose database has grown to a year of + # links reads one index entry per item now instead of the whole table per + # run. + def stored?(feed) + scope = model_class.where(unique_keys[0] => feed.link) + scope = scope.or(model_class.where(unique_keys[1] => feed.title)) if unique_keys.length > 1 + scope.exists? + end + def create_table - ActiveRecord::Migration.create_table(model_class.table_name) {|t| - column_definition.each_pair {|column_name, column_type| - t.column column_name, column_type - } - } + ActiveRecord::Base.connection.create_table(model_class.table_name) do |table| + column_definition.each_pair do |name, type| + table.column name, type + end + end end def db_dir - dir = (File.expand_path('~/.automatic/db')) - if File.directory?(dir) - dir - else - File.join(File.dirname(__FILE__), '..', '..', 'db') - end + dir = File.expand_path('~/.automatic/db') + return dir if File.directory?(dir) + + File.expand_path('../../db', __dir__) end def prepare_database - db = File.join(db_dir, @config['db']) - Automatic::Log.puts("info", "Using Database: #{db}") - ActiveRecord::Base.establish_connection( - :adapter => "sqlite3", - :database => db) - create_table unless model_class.table_exists? + db = File.join(db_dir, @config['db'].to_s) + Automatic::Log.puts('info', "Using Database: #{db}") + ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: db) + create_table unless ActiveRecord::Base.connection.table_exists?(model_class.table_name) end end end diff --git a/plugins/store/file.rb b/plugins/store/file.rb index 86bbba7..8456fe0 100644 --- a/plugins/store/file.rb +++ b/plugins/store/file.rb @@ -1,4 +1,3 @@ -#!/usr/bin/env ruby # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Store::File # Author: id774 (More info: http://id774.net) @@ -6,97 +5,122 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 28, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require 'fileutils' -require 'open-uri' require 'uri' module Automatic::Plugin class StoreFile + # A link with either scheme is fetched from S3 rather than over HTTP. + # `s3n` is what Recipes written for this plugin use; `s3` is the spelling + # everything else uses and is accepted as well. + S3_SCHEMES = %w[s3 s3n].freeze - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - # The AWS SDK is required here rather than at the top of the file, so - # that downloading over HTTP needs neither the gem nor a bucket. This - # branch is written against AWS SDK for Ruby v1 and needs rework for the - # current SDK; see doc/PLUGINS.md section 6.4. - unless @config['bucket_name'].nil? - require 'aws-sdk' - s3 = AWS::S3.new( - :access_key_id => @config['access_key'], - :secret_access_key => @config['secret_key'] - ) - @bucket = s3.buckets[@config['bucket_name']] - end - @return_feeds = [] end + # Downloads what each link points at and rewrites the link to a file URI, + # which is how PublishAmazonS3 later knows it has a local file. def run - @pipeline.each {|feeds| - unless feeds.nil? - feeds.items.each {|feed| - unless feed.link.nil? - Automatic::Log.puts("info", "Downloading File: #{feed.link}") - FileUtils.mkdir_p(@config['path']) unless FileTest.exist?(@config['path']) - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - retries += 1 - feed.link = get_file(feed.link) - sleep ||= @config['interval'].to_i - @return_feeds << feed - rescue - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault during file download.") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - end - } + stored = [] + @pipeline.each do |feeds| + next if feeds.nil? + + feeds.items.each do |feed| + next if feed.link.nil? + + stored << feed if store(feed) end - } + end + @pipeline = [] - @pipeline << Automatic::FeedMaker.create_pipeline(@return_feeds) if @return_feeds.length > 0 + @pipeline << Automatic::FeedMaker.create_pipeline(stored) unless stored.empty? @pipeline end private + def store(feed) + Automatic::Log.puts('info', "Downloading File: #{feed.link}") + FileUtils.mkdir_p(@config['path'].to_s) + + retries = 0 + retry_max = @config['retry'].to_i + begin + feed.link = get_file(feed.link) + sleep(@config['interval'].to_i) + true + rescue StandardError => e + retries += 1 + Automatic::Log.puts('error', + "ErrorCount: #{retries}, Fault during file download: #{e.message}") + return false if retries > retry_max + + sleep(@config['interval'].to_i) + retry + end + end + def get_file(url) uri = URI.parse(url) - case uri.scheme - when "s3n" - return_path = get_aws(uri) - else - return_path = wget(uri, url) - end - Automatic::Log.puts("info", "Saved File: #{return_path}") - "file://" + return_path + path = S3_SCHEMES.include?(uri.scheme) ? from_s3(uri) : download(url) + Automatic::Log.puts('info', "Saved File: #{path}") + "file://#{path}" end - def wget(uri, url) - filename = File.basename(uri.path) - filepath = File.join(@config['path'], filename) - URI.open(url) {|source| - File.open(filepath, "w+b") { |o| - o.print(source.read) - } - } - filepath + # Only HTTP and HTTPS are fetched: a link arrives from a feed, which is to + # say from outside, and `file://` is not something a store plugin should + # be talked into reading. + def download(url) + path = local_path(Automatic::Http.uri(url).path) + File.binwrite(path, Automatic::Http.read(url)) + path end - def get_aws(uri) - filename = File.basename(uri.path) - filepath = File.join(@config['path'], filename) - object = @bucket.objects[uri.path] - File.open(filepath, 'wb') do |file| - object.read do |chunk| - file.write(chunk) - end + # AWS SDK for Ruby v3, one gem for one service. The bucket is the Recipe's + # `bucket_name` where it has one, so that an existing Recipe keeps its + # meaning, and the link's own host otherwise. Credentials are the Recipe's + # where it carries them and the SDK's default chain -- environment, + # profile, instance role -- where it does not, which is the way to run + # this without a secret in a file. + def from_s3(uri) + path = local_path(uri.path) + s3.get_object(bucket: bucket(uri), key: uri.path.sub(%r{\A/}, ''), + response_target: path) + path + end + + def s3 + @s3 ||= begin + Automatic.require_optional('aws-sdk-s3', needed_by: 'the S3 path of StoreFile') + Aws::S3::Client.new(**client_options) + end + end + + def client_options + options = {} + options[:region] = @config['region'].to_s unless @config['region'].nil? + unless @config['access_key'].nil? + options[:access_key_id] = @config['access_key'].to_s + options[:secret_access_key] = @config['secret_key'].to_s end - filepath + options + end + + def bucket(uri) + name = @config['bucket_name'].to_s + name.empty? ? uri.host.to_s : name + end + + def local_path(remote_path) + name = File.basename(remote_path.to_s) + raise ArgumentError, "no file name in #{remote_path}" if name.empty? || name == '/' + + File.join(@config['path'].to_s, name) end end end diff --git a/plugins/store/full_text.rb b/plugins/store/full_text.rb index b70b782..9ea72ca 100644 --- a/plugins/store/full_text.rb +++ b/plugins/store/full_text.rb @@ -5,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 26, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require_relative 'database' @@ -17,43 +17,48 @@ class Blog < ActiveRecord::Base class StoreFullText include Automatic::Plugin::Database - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end def column_definition { - :title => :string, - :link => :string, - :description => :string, - :content => :string, - :created_at => :string, + title: :string, + link: :string, + description: :string, + content: :string, + created_at: :string } end + # Link or title, so that a republished article with a new URL is not + # stored twice. def unique_keys - [:link, :title] + %i[link title] end def model_class Automatic::Plugin::Blog end + # Records title, link, description and content_encoded, and passes on only + # what is new. def run - for_each_new_feed {|feed| - Automatic::Log.puts("info", "Saving FullText: #{feed.link}") + for_each_new_feed do |feed| + Automatic::Log.puts('info', "Saving FullText: #{feed.link}") begin Blog.create( - :title => feed.title, - :link => feed.link, - :description => feed.description, - :content => feed.content_encoded, - :created_at => Time.now.strftime("%Y/%m/%d %X")) - rescue - Automatic::Log.puts("warn", "Skip feed due to fault in save.") + title: feed.title, + link: feed.link, + description: feed.description, + content: feed.content_encoded, + created_at: Time.now.strftime('%Y/%m/%d %X') + ) + rescue StandardError => e + Automatic::Log.puts('warn', "Skip feed due to fault in save: #{e.message}") end - } + end end end end diff --git a/plugins/store/permalink.rb b/plugins/store/permalink.rb index c6c69db..8f29161 100644 --- a/plugins/store/permalink.rb +++ b/plugins/store/permalink.rb @@ -1,4 +1,3 @@ -#!/usr/bin/env ruby # -*- coding: utf-8 -*- # Name:: Automatic::Plugin::Store::Permalink # Author: id774 (More info: http://id774.net) @@ -6,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 22, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require_relative 'database' @@ -18,16 +17,13 @@ class Permalink < ActiveRecord::Base class StorePermalink include Automatic::Plugin::Database - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end def column_definition - { - :url => :string, - :created_at => :string - } + { url: :string, created_at: :string } end def unique_keys @@ -38,15 +34,13 @@ def model_class Automatic::Plugin::Permalink end + # Records each item's link and passes on only the links not already + # recorded. The usual guard against publishing the same item twice. def run - for_each_new_feed {|feed| - unless feed.link.nil? - Permalink.create( - :url => feed.link, - :created_at => Time.now.strftime("%Y/%m/%d %X")) - Automatic::Log.puts("info", "Saving Permalink: #{feed.link}") - end - } + for_each_new_feed do |feed| + Permalink.create(url: feed.link, created_at: Time.now.strftime('%Y/%m/%d %X')) + Automatic::Log.puts('info', "Saving Permalink: #{feed.link}") + end end end end diff --git a/plugins/subscription/chan_toru.rb b/plugins/subscription/chan_toru.rb deleted file mode 100644 index b876b6d..0000000 --- a/plugins/subscription/chan_toru.rb +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::ChanToru -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Jun 28, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require_relative 'g_guide' - -module Automatic::Plugin - class SubscriptionChanToru - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - end - - def g_guide_pipeline - SubscriptionGGuide.new(@config, @pipeline).run - end - - def run - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - pipeline = g_guide_pipeline - pipeline.each {|feeds| - feeds.items.each {|feed| - feed = link_change(feed) - } - } - @pipeline = pipeline - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{retries}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - - @pipeline - end - - def link_change(feed) - feed.link.gsub(/([0-9]+)/) do |pid| - if pid != '' - feed.link = "https://tv.so-net.ne.jp/chan-toru/intent" + - "?cat=1&area=23&pid=#{pid}&from=tw" - else - feed.link = nil - end - end - feed - end - - end -end diff --git a/plugins/subscription/feed.rb b/plugins/subscription/feed.rb index 696a1ff..0afa150 100644 --- a/plugins/subscription/feed.rb +++ b/plugins/subscription/feed.rb @@ -5,31 +5,42 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 22, 2012 -# Updated:: Feb 21, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class SubscriptionFeed - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end def run - @config['feeds'].each {|feed| - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - rss = Automatic::FeedParser.get_url(feed) - @pipeline << rss - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{feed}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - } + Array(@config['feeds']).each do |url| + feed = fetch(url) + @pipeline << feed unless feed.nil? + end @pipeline end + + private + + # A feed that fails after its retries is logged and skipped; the others + # still run. + def fetch(url) + retries = 0 + retry_max = @config['retry'].to_i + begin + Automatic::FeedParser.get_url(url) + rescue StandardError => e + retries += 1 + Automatic::Log.puts('error', + "ErrorCount: #{retries}, Fault in parsing: #{url}, #{e.message}") + return nil if retries > retry_max + + sleep(@config['interval'].to_i) + retry + end + end end end diff --git a/plugins/subscription/g_guide.rb b/plugins/subscription/g_guide.rb deleted file mode 100644 index 50551bd..0000000 --- a/plugins/subscription/g_guide.rb +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::GGuide -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Jun 28, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class SubscriptionGGuide - require 'uri' - - G_GUIDE_RSS = 'http://tv.so-net.ne.jp/rss/schedulesBySearch.action?' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - unless @config['keyword'].nil? || @config['keyword'].index(',').nil? - @keywords = @config['keyword'].split(',') - else - @keywords = [@config['keyword']] - end - end - - def run - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - @keywords.each {|keyword| - @pipeline << Automatic::FeedParser.get_url(feed_url keyword) - } - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{retries}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - @pipeline - end - - def feed_url keyword = nil - feed = G_GUIDE_RSS - unless keyword.nil? - feed += "condition.keyword=#{keyword}&" - end - feed += station_param - URI::RFC2396_Parser.new.escape(feed) - end - - def station_param - station = 0 - unless @config['station'].nil? - station = '1' if @config['station'] == '地上波' - end - "stationPlatformId=#{station}&" - end - end -end diff --git a/plugins/subscription/link.rb b/plugins/subscription/link.rb index caf8323..fa17ead 100644 --- a/plugins/subscription/link.rb +++ b/plugins/subscription/link.rb @@ -5,45 +5,42 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Sep 18, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class SubscriptionLink - require 'open-uri' - require 'rss' - - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Returns only what it fetched, discarding any incoming pipeline. def run - @return_feeds = [] - @config['urls'].each {|url| - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - create_rss(URI::RFC2396_Parser.new.escape(url)) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - } - @return_feeds + Array(@config['urls']).each_with_object([]) do |url, feeds| + rss = fetch(url) + feeds << rss unless rss.nil? + end end private - def create_rss(url) - Automatic::Log.puts("info", "Parsing Link: #{url}") - html = URI.open(url).read - unless html.nil? - rss = Automatic::FeedParser.parse_html(html) - sleep ||= @config['interval'].to_i - @return_feeds << rss + def fetch(url) + retries = 0 + retry_max = @config['retry'].to_i + begin + Automatic::Log.puts('info', "Parsing Link: #{url}") + rss = Automatic::FeedParser.parse_html(Automatic::Http.read(url)) + sleep(@config['interval'].to_i) + rss + rescue StandardError => e + retries += 1 + Automatic::Log.puts('error', + "ErrorCount: #{retries}, Fault in parsing: #{url}, #{e.message}") + return nil if retries > retry_max + + sleep(@config['interval'].to_i) + retry end end end diff --git a/plugins/subscription/pocket.rb b/plugins/subscription/pocket.rb deleted file mode 100644 index 5ce0f7f..0000000 --- a/plugins/subscription/pocket.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::Pocket -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 21, 2013 -# Updated:: Feb 21, 2014 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class SubscriptionPocket - require 'pocket' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - Pocket.configure do |c| - c.consumer_key = @config['consumer_key'] - c.access_token = @config['access_token'] - end - @client = Pocket.client - end - - def run - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - return_feeds = generate_feed(@client.retrieve(@config['optional'])) - @pipeline << Automatic::FeedMaker.create_pipeline(return_feeds) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{retries}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - @pipeline - end - - def generate_feed(retrieve) - return_feeds = [] - retrieve['list'].each {|key,list| - hashie = Hashie::Mash.new - hashie.title = list['given_title'] - hashie.link = list['given_url'] - hashie.description = list['excerpt'] - return_feeds << hashie - } - return_feeds - end - end -end diff --git a/plugins/subscription/text.rb b/plugins/subscription/text.rb index fb17570..ef8a353 100644 --- a/plugins/subscription/text.rb +++ b/plugins/subscription/text.rb @@ -5,64 +5,49 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: May 6, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class SubscriptionText - def initialize(config, pipeline=[]) - @config = config + # Columns of a TSV row, in order. A row with fewer columns leaves the rest + # unset, which is what makes a one-column file of titles a valid input. + COLUMNS = %w[title url description author comments].freeze + + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline - @return_feeds = [] end + # Reaches no network, which is what makes this the plugin to test a + # Recipe's later half with. Any combination of the four keys may be given. def run - create_feed - @pipeline << Automatic::FeedMaker.create_pipeline(@return_feeds) if @return_feeds.length > 0 + items = titles + urls + feeds + files + @pipeline << Automatic::FeedMaker.create_pipeline(items) unless items.empty? @pipeline end private - def create_feed - unless @config.nil? - @dummyfeeds = [] - unless @config['titles'].nil? - @config['titles'].each {|title| - feed = {} - feed['title'] = title - @return_feeds << Automatic::FeedMaker.generate_feed(feed) - } - end + def titles + Array(@config['titles']).map { |title| Automatic::FeedMaker.generate_feed('title' => title) } + end - unless @config['urls'].nil? - @config['urls'].each {|url| - feed = {} - feed['url'] = url - @return_feeds << Automatic::FeedMaker.generate_feed(feed) - } - end + def urls + Array(@config['urls']).map { |url| Automatic::FeedMaker.generate_feed('url' => url) } + end - unless @config['feeds'].nil? - @config['feeds'].each {|feed| - @return_feeds << Automatic::FeedMaker.generate_feed(feed) - } - end + def feeds + Array(@config['feeds']).map { |feed| Automatic::FeedMaker.generate_feed(feed) } + end - unless @config['files'].nil? - @config['files'].each {|f| - File.open(File.expand_path(f)) do |file| - file.each_line do |line| - feed = {} - feed['title'], feed['url'], feed['description'], feed['author'], - feed['comments'] = line.force_encoding("utf-8").strip.split("\t") - @return_feeds << Automatic::FeedMaker.generate_feed(feed) - end - end - } + # Tab separated, read as UTF-8, and `~` expanded. + def files + Array(@config['files']).flat_map do |path| + File.foreach(File.expand_path(path), encoding: 'UTF-8').map do |line| + Automatic::FeedMaker.generate_feed(COLUMNS.zip(line.strip.split("\t")).to_h) end end end - end end diff --git a/plugins/subscription/tumblr.rb b/plugins/subscription/tumblr.rb index 2f114eb..da93997 100644 --- a/plugins/subscription/tumblr.rb +++ b/plugins/subscription/tumblr.rb @@ -5,57 +5,66 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Oct 16, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class SubscriptionTumblr - require 'open-uri' - - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end + # Reads HTML written for a browser, so what it finds depends on the theme + # a given blog uses. Verify against the blog you mean to follow before + # putting it in cron, and set `interval`. def run - @return_feeds = [] - @config['urls'].each {|url| - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - create_rss(url) - unless @config['pages'].nil? - @config['pages'].times {|i| - if i > 0 - old_url = url + "/page/" + (i+1).to_s - create_rss(old_url) - end - } - end - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max + Array(@config['urls']).each_with_object([]) do |url, feeds| + pages(url).each do |page| + rss = fetch(page, url) + feeds << rss unless rss.nil? end - } - @return_feeds + end end private - def create_rss(url) - Automatic::Log.puts("info", "Parsing Tumblr: #{url}") - html = URI.open(url).read - unless html.nil? - uri = URI.parse(url) - rss = Automatic::FeedParser.parse_html(html) - rss.items.each {|item| - unless item.link =~ Regexp.new(uri.host) - item.link = nil - end - } - sleep ||= @config['interval'].to_i - @return_feeds << rss + + # The blog's own page, then /page/2 and onward. + def pages(url) + count = @config['pages'].to_i + return [url] if count < 2 + + [url] + (2..count).map { |number| "#{url}/page/#{number}" } + end + + def fetch(url, blog_url) + retries = 0 + retry_max = @config['retry'].to_i + begin + Automatic::Log.puts('info', "Parsing Tumblr: #{url}") + rss = Automatic::FeedParser.parse_html(Automatic::Http.read(url)) + drop_offsite_links(rss, blog_url) + sleep(@config['interval'].to_i) + rss + rescue StandardError => e + retries += 1 + Automatic::Log.puts('error', + "ErrorCount: #{retries}, Fault in parsing: #{url}, #{e.message}") + return nil if retries > retry_max + + sleep(@config['interval'].to_i) + retry + end + end + + # A theme's page carries the blog's own posts and a great deal else. A + # link that leaves the blog's host is blanked rather than removed, which + # is the pipeline's way of saying "not applicable"; the plugins after this + # one skip an item whose link is nil. + def drop_offsite_links(rss, blog_url) + host = Automatic::Http.uri(blog_url).host.to_s + rss.items.each do |item| + item.link = nil unless item.link.to_s.include?(host) end end end diff --git a/plugins/subscription/twitter.rb b/plugins/subscription/twitter.rb deleted file mode 100644 index 138f334..0000000 --- a/plugins/subscription/twitter.rb +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::Twitter -# Author: id774 (More info: http://id774.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Sep 9, 2012 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class SubscriptionTwitter - require 'open-uri' - Automatic.require_optional('nokogiri', needed_by: 'SubscriptionTwitter') - require 'rss' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - end - - def run - @return_feeds = [] - @config['urls'].each {|url| - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - create_rss(url) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - } - @return_feeds - end - - private - - def create_rss(url) - Automatic::Log.puts("info", "Parsing Twitter: #{url}") - html = URI.open(url).read - unless html.nil? - rss = RSS::Maker.make("2.0") {|maker| - xss = maker.xml_stylesheets.new_xml_stylesheet - xss.href = "http://twitter.com" - maker.channel.about = "http://twitter.com/index.rdf" - maker.channel.title = "Twitter" - maker.channel.description = "Twitter" - maker.channel.link = "http://twitter.com/" - maker.items.do_sort = true - doc = Nokogiri::HTML(html) - doc.xpath("/html/body/div").search('[@class="content"]').each {|content| - item = maker.items.new_item - item.title = content.search('[@class="username js-action-profile-name"]').text.to_s - content.search('[@class="tweet-timestamp js-permalink js-nav"]').each {|node| - item.link = "http://twitter.com" + node['href'].to_s - } - content.search('[@class="_timestamp js-short-timestamp js-relative-timestamp"]').each {|node| - item.date = Time.at(node['data-time'].to_i) - } - item.description = content.search('[@class="js-tweet-text"]').text.to_s - } - } - sleep ||= @config['interval'].to_i - @return_feeds << rss - end - end - end -end diff --git a/plugins/subscription/twitter_search.rb b/plugins/subscription/twitter_search.rb deleted file mode 100644 index 316511d..0000000 --- a/plugins/subscription/twitter_search.rb +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::TwitterSearch -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 30, 2013 -# Updated:: Feb 21, 2014 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class SubscriptionTwitterSearch - require 'twitter' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - @client = Twitter::Client.new( - :consumer_key => @config['consumer_key'], - :consumer_secret => @config['consumer_secret'], - :oauth_token => @config['oauth_token'], - :oauth_token_secret => @config['oauth_token_secret'] - ) - end - - def run - @pipeline = [] - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - return_feeds = [] - @client.search(@config['search'],@config['opt']).results.each do |status| - hashie = Hashie::Mash.new - hashie.title = 'Twitter Search' - hashie.link = "https://twitter.com/#{status.user['screen_name']}/status/#{status.id}" - hashie.description = status.text - hashie.author = status.user['screen_name'] - hashie.date = status.created_at - return_feeds << hashie - end - @pipeline << Automatic::FeedMaker.create_pipeline(return_feeds) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - @pipeline - end - end -end diff --git a/plugins/subscription/weather.rb b/plugins/subscription/weather.rb deleted file mode 100644 index 400bf72..0000000 --- a/plugins/subscription/weather.rb +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::Weather -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 12, 2013 -# Updated:: Feb 21, 2014 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -module Automatic::Plugin - class SubscriptionWeather - require 'weather_hacker' - - def initialize(config, pipeline=[]) - @config = config - @pipeline = pipeline - @weather = WeatherHacker.new(@config['zipcode']) - @day = 'today' - @day = @config['day'] unless @config['day'].nil? - end - - def run - weather = @weather.send(@day)['weather'] unless @weather.send(@day).nil? - if weather != nil - hashie = Hashie::Mash.new - hashie.title = weather - hashie.link = 'http://weather.dummy.' + @day - @pipeline << Automatic::FeedMaker.create_pipeline([hashie]) - end - @pipeline - end - end -end diff --git a/plugins/subscription/xml.rb b/plugins/subscription/xml.rb index 0b03f19..fc623c0 100644 --- a/plugins/subscription/xml.rb +++ b/plugins/subscription/xml.rb @@ -5,51 +5,53 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jul 12, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class SubscriptionXml require 'active_support/core_ext/hash/conversions' - require 'active_support/json' - require 'open-uri' - require 'rss' - require 'uri' + require 'json' - def initialize(config, pipeline=[]) - @config = config + def initialize(config, pipeline = []) + @config = config || {} @pipeline = pipeline end def run - @return_feeds = [] - @config['urls'].each {|url| - retries = 0 - retry_max = @config['retry'].to_i || 0 - begin - create_rss(URI::RFC2396_Parser.new.escape(url)) - rescue - retries += 1 - Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{url}") - sleep ||= @config['interval'].to_i - retry if retries <= retry_max - end - } - @return_feeds + Array(@config['urls']).each_with_object([]) do |url, feeds| + rss = fetch(url) + feeds << rss unless rss.nil? + end end private - def create_rss(url) - Automatic::Log.puts("info", "Parsing XML: #{url}") - hash = Hash.from_xml(URI.open(url).read) - json = hash.to_json - data = ActiveSupport::JSON.decode(json) - unless data.nil? - rss = Automatic::FeedMaker.content_provide(url, data) - sleep ||= @config['interval'].to_i - @return_feeds << rss + def fetch(url) + retries = 0 + retry_max = @config['retry'].to_i + begin + Automatic::Log.puts('info', "Parsing XML: #{url}") + rss = Automatic::FeedMaker.content_provide(url, document(url)) + sleep(@config['interval'].to_i) + rss + rescue StandardError => e + retries += 1 + Automatic::Log.puts('error', + "ErrorCount: #{retries}, Fault in parsing: #{url}, #{e.message}") + return nil if retries > retry_max + + sleep(@config['interval'].to_i) + retry end end + + # The document as plain hashes, arrays and strings. The round trip through + # JSON is what flattens what Hash.from_xml returns -- dates, times and + # ActiveSupport's own string subclasses -- into the values a consumer such + # as ProvideFluentd can serialize. + def document(url) + JSON.parse(Hash.from_xml(Automatic::Http.read(url)).to_json) + end end end diff --git a/script/build b/script/build index d8210b8..2877589 100755 --- a/script/build +++ b/script/build @@ -4,8 +4,9 @@ # Local build script # # Runs what CI runs, and optionally the integration recipes, which CI -# does not run and never will: they reach real services, several of -# which no longer exist. See doc/PLUGINS.md section 6. +# does not run and never will: they reach real services, and most need +# a credential or something the operator runs. See doc/PLUGINS.md +# section 6. # # Usage: # script/build Run the test suite and the CLI checks. @@ -16,6 +17,8 @@ # License: The GPL version 3, or LGPL version 3 (Dual License). # Contact: idnanashi@gmail.com # +# v2.1 8/15,2026 +# Drop the recipes of the plugins removed in v26.08. # v2.0 8/14,2026 # Replace the Jenkins-era workflow: drop bundle update and the # simplecov rake task, use the -v option the CLI now has, and @@ -53,8 +56,8 @@ unit_tests() { } # The integration recipes reach real services and are run deliberately. -# Most need a credential, a service that has shut down, or both; read the -# recipe before running it. +# Most need a credential or a service the operator runs; read the recipe +# before running it. integration_tests() { for recipe in "$ROOT_DIR"/test/integration/test_*.yml do diff --git a/spec/doc/plugins_catalogue_spec.rb b/spec/doc/plugins_catalogue_spec.rb new file mode 100644 index 0000000..9fbca41 --- /dev/null +++ b/spec/doc/plugins_catalogue_spec.rb @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# Name:: The plugin catalogue +# Author: id774 (More info: http://id774.net) +# Source Code:: https://github.com/id774/automaticruby +# License:: The GPL version 3, or LGPL version 3 (Dual License). +# Contact:: idnanashi@gmail.com +# Created:: Aug 15, 2026 +# Updated:: Aug 15, 2026 +# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. +# +# doc/PLUGINS.md section 6 is the catalogue of what ships, and README.md +# repeats its counts. Both are prose, and prose drifts from a directory. This +# holds them to it: a plugin with no entry, an entry with no plugin, and a +# count left behind by an edit are failures of the ordinary suite rather than +# something a reader discovers. + +require File.expand_path(File.join(File.dirname(__FILE__), '../spec_helper')) + +RSpec.describe 'the plugin catalogue' do + # Every plugin class the gem ships, from the files themselves. A file that + # defines no plugin class -- store/database.rb, which is the storage the + # store plugins share -- is not one and is named here. + SHARED_FILES = ['store/database.rb'].freeze + + # A plugin file may define a helper class beside the plugin itself, so the + # plugin is the class whose name maps back to the file the loader would find + # it in -- which is the rule of section 3.2 applied in the other direction. + def shipped_plugins + Dir[File.join(APP_ROOT, 'plugins', '*', '*.rb')].flat_map { |path| + relative = path.sub("#{File.join(APP_ROOT, 'plugins')}/", '') + next [] if SHARED_FILES.include?(relative) + + expected = relative.sub(/\.rb\z/, '').tr('/', '_') + names = File.read(path, encoding: 'UTF-8').scan(/^\s*class\s+([A-Z]\w*)/).flatten + found = names.select { |name| name.underscore == expected } + raise "#{relative} defines no class the loader would find" if found.empty? + + found + }.sort + end + + # "#### SubscriptionFeed — **Supported**" + ENTRY = /^\#\#\#\#\s+(\w+)\s+—\s+\*\*(.+?)\*\*/ + + def catalogue + document = File.read(File.join(APP_ROOT, 'doc', 'PLUGINS.md'), encoding: 'UTF-8') + section = document[/^## 6\. The plugins$.*?^## 7\./m] + raise 'doc/PLUGINS.md has no section 6' if section.nil? + + section.scan(ENTRY).to_h + end + + # "| Supported (external) | 10 | `SubscriptionTumblr`, ... |" + def summary_rows + document = File.read(File.join(APP_ROOT, 'doc', 'PLUGINS.md'), encoding: 'UTF-8') + document.scan(/^\|\s(Supported|Supported \(external\)|Needs rework)\s\|\s(\d+)\s\|\s(.+?)\s\|$/) + end + + let(:shipped) { shipped_plugins } + let(:entries) { catalogue } + + # The loader resolves a name against the installation root, which a spec run + # has not set. + around do |example| + root = Automatic.root_dir + Automatic.root_dir = APP_ROOT + example.run + Automatic.root_dir = root + end + + it 'has an entry for every plugin that ships' do + (shipped - entries.keys).should == [] + end + + it 'ships a plugin for every entry, at its loader-derived path' do + (entries.keys - shipped).should == [] + + entries.each_key do |name| + lambda { Automatic::Pipeline.load_plugin(name) }.should_not raise_error + end + end + + it 'gives every plugin one of the statuses section 5 defines' do + entries.each_pair do |name, status| + status.sub(/,.*\z/, '').should satisfy { |value| + ['Supported', 'Supported (external)', 'Needs rework'].include?(value) + }, "#{name} has the status #{status.inspect}" + end + end + + describe 'the summary table in section 7' do + it 'counts what section 6 lists' do + counted = entries.values.group_by { |status| status.sub(/,.*\z/, '') }. + transform_values(&:size) + + summary_rows.each do |status, count, _plugins| + count.to_i.should == counted.fetch(status, 0) + end + end + + it 'names what section 6 lists' do + summary_rows.each do |status, _count, plugins| + named = plugins.scan(/`(\w+)`/).flatten.sort + expected = entries.select { |_name, value| value.sub(/,.*\z/, '') == status }.keys.sort + named.should == expected + end + end + + it 'accounts for every plugin exactly once' do + summary_rows.sum { |_status, count, _plugins| count.to_i }.should == shipped.size + end + end + + describe 'README.md' do + let(:readme) { File.read(File.join(APP_ROOT, 'README.md'), encoding: 'UTF-8') } + + it 'gives the same total' do + readme.should include("#{shipped.size} plugins") + end + + it 'gives the same count per status' do + summary_rows.each do |status, count, _plugins| + readme.should match(/\*\*#{Regexp.escape(status)}\*\*\s*\|\s*#{count}\s*\|/) + end + end + end +end diff --git a/spec/lib/automatic/http_spec.rb b/spec/lib/automatic/http_spec.rb new file mode 100644 index 0000000..6251c50 --- /dev/null +++ b/spec/lib/automatic/http_spec.rb @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Http +# Author: id774 (More info: http://id774.net) +# Source Code:: https://github.com/id774/automaticruby +# License:: The GPL version 3, or LGPL version 3 (Dual License). +# Contact:: idnanashi@gmail.com +# Created:: Aug 15, 2026 +# Updated:: Aug 15, 2026 +# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. + +require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') + +describe Automatic::Http do + describe '.uri' do + it 'parses an ordinary URL' do + Automatic::Http.uri('https://example.com/feed').to_s.should == 'https://example.com/feed' + end + + it 'normalizes the host and an empty path' do + Automatic::Http.uri('http://EXAMPLE.com').to_s.should == 'http://example.com/' + end + + # A feed URL carrying a Japanese query term, or a link with a space in it, + # is escaped and parsed again rather than ending the run. + it 'escapes a string carrying characters a URI may not' do + Automatic::Http.uri('http://example.com/a b').to_s.should == 'http://example.com/a%20b' + end + + it 'trims surrounding whitespace' do + Automatic::Http.uri(" https://example.com/feed\n").to_s. + should == 'https://example.com/feed' + end + + # A link arrives from a feed, which is to say from outside. Only HTTP and + # HTTPS are fetched, so that a plugin cannot be talked into reading a local + # file or opening an FTP session. + %w[file:///etc/passwd ftp://example.com/x gopher://example.com/].each do |url| + it "refuses #{url}" do + lambda { Automatic::Http.uri(url) }. + should raise_error(ArgumentError, /not an HTTP or HTTPS URL/) + end + end + + it 'refuses an empty URL' do + lambda { Automatic::Http.uri(' ') }.should raise_error(ArgumentError, /no URL/) + end + + it 'refuses a string that is not a URL at all' do + lambda { Automatic::Http.uri('invalid_url') }.should raise_error(ArgumentError) + end + end + + describe '.fetchable?' do + it 'answers for a URL this framework will fetch' do + Automatic::Http.fetchable?('https://example.com/').should be true + end + + it 'answers false rather than raising for one it will not' do + Automatic::Http.fetchable?('file:///etc/passwd').should be false + Automatic::Http.fetchable?(nil).should be false + end + end + + describe '.read' do + it 'opens the URI with a timeout, a redirect limit and this project as the agent' do + uri = double('uri') + Automatic::Http.stub(:uri).and_return(uri) + uri.should_receive(:open) { |options, &block| + options['User-Agent'].should include('Automatic Ruby') + options[:open_timeout].should == Automatic::Http::OPEN_TIMEOUT + options[:read_timeout].should == Automatic::Http::READ_TIMEOUT + options[:max_redirects].should == Automatic::Http::REDIRECT_LIMIT + block.call(StringIO.new('a body')) + } + + Automatic::Http.read('https://example.com/').should == 'a body' + end + end +end diff --git a/spec/plugins/custom_feed/svn_log_spec.rb b/spec/plugins/custom_feed/svn_log_spec.rb index 2e9fbaa..a699493 100644 --- a/spec/plugins/custom_feed/svn_log_spec.rb +++ b/spec/plugins/custom_feed/svn_log_spec.rb @@ -1,38 +1,111 @@ # -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::CustomFeed::SVNFLog +# Name:: Automatic::Plugin::CustomFeed::SVNLog # Author: kzgs # Source Code:: https://github.com/id774/automaticruby # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 29, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# CustomFeedSVNLog needs the optional xml-simple gem and the svn command; -# doc/PLUGINS.md section 6.2 classifies it as Supported (external). -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('custom_feed/svn_log') +require 'custom_feed/svn_log' +# CustomFeedSVNLog needs the svn command, which is the operator's to install +# and which doc/PLUGINS.md section 6.2 classifies it as Supported (external) +# for. What is verified here is everything on this side of that command: the +# document it prints becomes a feed, and the arguments it is given are the +# ones the plugin's settings ask for. Running the command itself is the +# :network example at the end, which is excluded from the default suite. describe Automatic::Plugin::CustomFeedSVNLog do - context "with feeds whose valid URL" do + SVN_LOG_XML = <<~XML + + + + someone + 2026-08-01T09:15:22.123456Z + Fix the thing + + + another + 2026-07-30T22:04:11.000000Z + Add the other thing + + + XML + + subject { + Automatic::Plugin::CustomFeedSVNLog.new( + 'target' => 'https://svn.example.com/repos/project/', + 'fetch_items' => 2, + 'title' => 'project' + ) + } + + before { subject.stub(:svn_log).and_return(SVN_LOG_XML) } + + its(:run) { should have(1).feed } + + it 'makes one item per revision' do + subject.run[0].items.should have(2).items + end + + it 'titles an item with its message and author' do + subject.run[0].items[0].title.should == 'Fix the thing by someone' + end + + it 'links an item to its revision, with the trailing slash of target removed' do + subject.run[0].items[0].link. + should == 'https://svn.example.com/repos/project/!svn/bc/1913406' + end + + it 'takes the revision date' do + subject.run[0].items[0].date.should == Time.parse('2026-08-01T09:15:22.123456Z') + end + + it 'takes the channel title from the settings' do + subject.run[0].channel.title.should == 'project' + end + + context 'with no revisions' do + before { subject.stub(:svn_log).and_return("\n\n\n") } + + it 'returns the pipeline unchanged' do + subject.run.should be_empty + end + end + + describe 'the command' do + # An argument vector rather than a command line: a repository URL is an + # argument to svn and cannot become part of a shell command. + it 'passes the repository, --xml and the limit to svn' do + plugin = Automatic::Plugin::CustomFeedSVNLog.new( + 'target' => 'https://svn.example.com/repos/project', 'fetch_items' => 5 + ) + IO.should_receive(:popen). + with(['svn', 'log', 'https://svn.example.com/repos/project', + '--xml', '--limit=5'], err: File::NULL). + and_return(SVN_LOG_XML) + plugin.stub(:raise) + plugin.send(:revisions) + end + + it 'defaults the limit to 30' do + plugin = Automatic::Plugin::CustomFeedSVNLog.new( + 'target' => 'https://svn.example.com/repos/project' + ) + plugin.send(:limit).should == 30 + end + end + + context 'against a real repository', :network do subject { Automatic::Plugin::CustomFeedSVNLog.new( - { - 'target' => 'http://svn.apache.org/repos/asf/', - 'fetch_items' => 2 - }) + 'target' => 'https://svn.apache.org/repos/asf/', 'fetch_items' => 2 + ) } its(:run) { should have(1).feed } - - specify { - feed = subject.run[0] - feed.should have(2).items - } end end - diff --git a/spec/plugins/filter/absolute_uri_spec.rb b/spec/plugins/filter/absolute_uri_spec.rb index f335e99..503e62c 100644 --- a/spec/plugins/filter/absolute_uri_spec.rb +++ b/spec/plugins/filter/absolute_uri_spec.rb @@ -36,26 +36,26 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items[0].link. + returned = subject.run + returned[0].items[0].link. should == "http://id774.net/images/link_1.jpg" - subject.instance_variable_get(:@return_feeds)[0].items[1].link. + returned[0].items[1].link. should == "http://id774.net/images/link_2.jpg" - subject.instance_variable_get(:@return_feeds)[0].items[2].link. + returned[0].items[2].link. should == "http://id774.net/images/link_3.JPG" - subject.instance_variable_get(:@return_feeds)[0].items[3].link. + returned[0].items[3].link. should == "http://id774.net/images/link_4.png" - subject.instance_variable_get(:@return_feeds)[0].items[4].link. + returned[0].items[4].link. should == "http://id774.net/images/link_5.jpeg" - subject.instance_variable_get(:@return_feeds)[0].items[5].link. + returned[0].items[5].link. should == "http://id774.net/images/link_6.PNG" - subject.instance_variable_get(:@return_feeds)[0].items[6].link. + returned[0].items[6].link. should == "http://id774.net/images/link_8.gif" - subject.instance_variable_get(:@return_feeds)[0].items[7].link. + returned[0].items[7].link. should == "http://id774.net/images/link_9.GIF" - subject.instance_variable_get(:@return_feeds)[0].items[8].link. + returned[0].items[8].link. should == "http://id774.net/images/link_10.tiff" - subject.instance_variable_get(:@return_feeds)[0].items[9].link. + returned[0].items[9].link. should == "http://id774.net/images/link_11.TIFF" } end diff --git a/spec/plugins/filter/description_link_spec.rb b/spec/plugins/filter/description_link_spec.rb index 06cac9a..e34cac1 100644 --- a/spec/plugins/filter/description_link_spec.rb +++ b/spec/plugins/filter/description_link_spec.rb @@ -10,11 +10,11 @@ require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# FilterDescriptionLink needs the nkf and nokogiri gems, which the Gemfile +# FilterDescriptionLink reads a fetched page with nokogiri, which the Gemfile # declares in its optional :plugins group. The default suite and CI do not -# install them, so this spec runs only where the operator has. See +# install it, so this spec runs only where the operator has. See # doc/POLICY.md section 5. -if AutomaticSpec.optional_dependency?('nkf') && AutomaticSpec.optional_dependency?('nokogiri') +if AutomaticSpec.optional_dependency?('nokogiri') require 'filter/description_link' describe Automatic::Plugin::FilterDescriptionLink do @@ -142,5 +142,31 @@ end end + # The framework hands a plugin a Hashie::Mash, not a Hash. This tested the + # mapping's class and so read neither setting in any real run; the spec + # below is the one that would have caught it. + context "with the mapping a Recipe actually produces" do + + subject { + Automatic::Plugin::FilterDescriptionLink.new( + Hashie::Mash.new('clear_description' => 1), + AutomaticSpec.generate_pipeline { + feed { + item "http://test1.id774.net", + "dummy title", + "aaa bbb ccc http://test2.id774.net ddd eee", + "Mon, 07 Mar 2011 15:54:11 +0900" + } + } + ) + } + + specify { + returned = subject.run + returned[0].items[0].link.should == "http://test2.id774.net" + returned[0].items[0].description.should == "" + } + end + end end diff --git a/spec/plugins/filter/github_feed_spec.rb b/spec/plugins/filter/github_feed_spec.rb index 0082c69..f2109fb 100644 --- a/spec/plugins/filter/github_feed_spec.rb +++ b/spec/plugins/filter/github_feed_spec.rb @@ -35,20 +35,20 @@ its(:run) { should have(1).feeds } specify { - subject.run + returned = subject.run - subject.instance_variable_get(:@return_feeds)[0].items[0].link + returned[0].items[0].link .should == '1' - subject.instance_variable_get(:@return_feeds)[0].items[0].title + returned[0].items[0].title .should == 'title1' - subject.instance_variable_get(:@return_feeds)[0].items[0].description + returned[0].items[0].description .should == 'description1' - subject.instance_variable_get(:@return_feeds)[0].items[1].link + returned[0].items[1].link .should == '0' - subject.instance_variable_get(:@return_feeds)[0].items[1].title + returned[0].items[1].title .should == 'title0' - subject.instance_variable_get(:@return_feeds)[0].items[1].description + returned[0].items[1].description .should == 'description0' } end diff --git a/spec/plugins/filter/google_news_spec.rb b/spec/plugins/filter/google_news_spec.rb deleted file mode 100644 index 973e1fb..0000000 --- a/spec/plugins/filter/google_news_spec.rb +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Filter::GoogleNews -# Author: id774 (More info: http://id774.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Oct 12, 2014 -# Updated:: Oct 12, 2014 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -require 'filter/google_news' - -describe Automatic::Plugin::FilterGoogleNews do - - context "It should be not rewrite link other urls" do - - subject { - Automatic::Plugin::FilterGoogleNews.new( - {}, - AutomaticSpec.generate_pipeline { - feed { - item "http://test1.id774.net", - "dummy title", - "aaa bbb ccc http://test2.id774.net ddd eee", - "Mon, 07 Mar 2011 15:54:11 +0900" - } - } - ) - } - - describe "#run" do - its(:run) { should have(1).feeds } - - specify { - subject.run - subject.instance_variable_get(:@pipeline)[0].items[0].link. - should == "http://test1.id774.net" - } - end - end - - context "It should be rewrite link for google news urls" do - - subject { - Automatic::Plugin::FilterGoogleNews.new( - {}, - AutomaticSpec.generate_pipeline { - feed { - item "http://news.google.com/news/url?sa=t&fd=R&ct2=us&usg=AFQjCNGhIFo1illQ6jFyVGPZtfkttFaJYQ&clid=c3a7d30bb8a4878e06b80cf16b898331&cid=52779138313507&ei=Pts3VLDAD4X7kgWO9oGIBg&url=http://www.yomiuri.co.jp/world/20141010-OYT1T50090.html", - "dummy title", - "aaa bbb ccc http://test2.id774.net ddd eee", - "Mon, 07 Mar 2011 15:54:11 +0900" - } - } - ) - } - - describe "#run" do - its(:run) { should have(1).feeds } - - specify { - subject.run - subject.instance_variable_get(:@pipeline)[0].items[0].link. - should == "http://www.yomiuri.co.jp/world/20141010-OYT1T50090.html" - } - end - end - -end diff --git a/spec/plugins/filter/image_source_spec.rb b/spec/plugins/filter/image_source_spec.rb index 132f762..58fae8f 100644 --- a/spec/plugins/filter/image_source_spec.rb +++ b/spec/plugins/filter/image_source_spec.rb @@ -30,8 +30,8 @@ describe "#run" do its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items[0].link. + returned = subject.run + returned[0].items[0].link. should == "http://27.media.tumblr.com/tumblr_lzrubkfPlt1qb8vzto1_500.png" } end @@ -51,8 +51,8 @@ describe "#run" do its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items[0].link. + returned = subject.run + returned[0].items[0].link. should == "http://24.media.tumblr.com/tumblr_m07wttnIdy1qzoj1jo1_400.jpg" } end @@ -69,27 +69,68 @@ "" }})} + # The page behind the link is read through Automatic::Http, which is this + # framework's own boundary rather than a service being simulated: the spec + # says what the page contains and asserts on what the plugin makes of it. describe "#run" do before do - subject.stub(:rewrite_link).and_return(['http://huge.png']) + Automatic::Http.stub(:read). + and_return('') end its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items[0].link. + returned = subject.run + returned[0].items[0].link. should == 'http://huge.png' } end - describe "#imgs" do + describe "with several images on the page" do before do - response = Hashie::Mash.new - response.read = '
' - URI.stub(:open).and_return(response) + Automatic::Http.stub(:read). + and_return('
') end its(:run) { subject.run[0].items.length.should == 2 } end + + describe "with a page whose images are relative" do + before do + Automatic::Http.stub(:read). + and_return("") + end + + specify { + subject.run[0].items.map(&:link).sort. + should == ['http://tumblr.com/a.png', 'http://tumblr.com/b.png'] + } + end + + describe "when the page cannot be read" do + before do + Automatic::Http.stub(:read).and_raise(StandardError, 'no such host') + end + + its(:run) { should have(1).feeds } + specify { subject.run[0].items.should be_empty } + end + end +end + +describe Automatic::Plugin::FilterImageSource do + context "with a description quoting src with apostrophes" do + subject { + Automatic::Plugin::FilterImageSource.new({}, + AutomaticSpec.generate_pipeline { + feed { + item "http://example.com/post", "", + "

x

" + }})} + + specify { + subject.run[0].items.map(&:link). + should == ['http://example.com/a.png'] + } end end diff --git a/spec/plugins/filter/image_spec.rb b/spec/plugins/filter/image_spec.rb index d4b5ecd..250522d 100644 --- a/spec/plugins/filter/image_spec.rb +++ b/spec/plugins/filter/image_spec.rb @@ -36,32 +36,62 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items[0].link. + returned = subject.run + returned[0].items[0].link. should == "http://id774.net/images/link_1.jpg" - subject.instance_variable_get(:@return_feeds)[0].items[1].link. + returned[0].items[1].link. should == "http://id774.net/images/link_2.jpg" - subject.instance_variable_get(:@return_feeds)[0].items[2].link. + returned[0].items[2].link. should == "http://id774.net/images/link_3.JPG" - subject.instance_variable_get(:@return_feeds)[0].items[3].link. + returned[0].items[3].link. should == "http://id774.net/images/link_4.png" - subject.instance_variable_get(:@return_feeds)[0].items[4].link. + returned[0].items[4].link. should == "http://id774.net/images/link_5.jpeg" - subject.instance_variable_get(:@return_feeds)[0].items[5].link. + returned[0].items[5].link. should == "http://id774.net/images/link_6.PNG" - subject.instance_variable_get(:@return_feeds)[0].items[6].link. + returned[0].items[6].link. should be_nil - subject.instance_variable_get(:@return_feeds)[0].items[7].link. + returned[0].items[7].link. should == "http://id774.net/images/link_8.gif" - subject.instance_variable_get(:@return_feeds)[0].items[8].link. + returned[0].items[8].link. should == "http://id774.net/images/link_9.GIF" - subject.instance_variable_get(:@return_feeds)[0].items[9].link. + returned[0].items[9].link. should == "http://id774.net/images/link_10.tiff" - subject.instance_variable_get(:@return_feeds)[0].items[10].link. + returned[0].items[10].link. should be_nil - subject.instance_variable_get(:@return_feeds)[0].items[11].link. + returned[0].items[11].link. should == "http://id774.net/images/link_11.TIFF" } end end end + +describe Automatic::Plugin::FilterImage do + # The test is on the path rather than on the whole link, so that an image + # served with a query string -- which is how most of what serves images now + # serves them -- is recognised. webp and avif are images too. + context "with links of the shapes the current web serves" do + subject { + Automatic::Plugin::FilterImage.new({}, + AutomaticSpec.generate_pipeline { + feed { + item "https://example.com/a.jpg?w=1280&v=2" + item "https://example.com/b.webp" + item "https://example.com/c.avif" + item "https://example.com/d.tif" + item "https://example.com/e.html?image=f.jpg" + item "https://example.com/" + }})} + + specify { + subject.run[0].items.map(&:link).should == [ + "https://example.com/a.jpg?w=1280&v=2", + "https://example.com/b.webp", + "https://example.com/c.avif", + "https://example.com/d.tif", + nil, + nil + ] + } + end +end diff --git a/spec/plugins/filter/one_spec.rb b/spec/plugins/filter/one_spec.rb index 438136b..e1f445d 100644 --- a/spec/plugins/filter/one_spec.rb +++ b/spec/plugins/filter/one_spec.rb @@ -32,10 +32,10 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items. + returned = subject.run + returned[0].items. count.should == 1 - subject.instance_variable_get(:@return_feeds)[0].items[0].link. + returned[0].items[0].link. should == 'http://aaa.png' } end @@ -62,10 +62,10 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items. + returned = subject.run + returned[0].items. count.should == 1 - subject.instance_variable_get(:@return_feeds)[0].items[0].link. + returned[0].items[0].link. should == 'http://ddd.png' } end diff --git a/spec/plugins/filter/rand_spec.rb b/spec/plugins/filter/rand_spec.rb index 3718ba6..f6ae740 100644 --- a/spec/plugins/filter/rand_spec.rb +++ b/spec/plugins/filter/rand_spec.rb @@ -38,8 +38,8 @@ def rand_plugin # outcome and fell through to a pending block on the one permutation in # twenty-four where nothing moved, so it failed at random. specify "returns exactly the input items" do - subject.run - links = subject.instance_variable_get(:@return_feeds)[0].items.map(&:link) + returned = subject.run + links = returned[0].items.map(&:link) links.sort.should == LINKS.sort end diff --git a/spec/plugins/filter/sanitize_spec.rb b/spec/plugins/filter/sanitize_spec.rb index 189e15b..f2d177b 100644 --- a/spec/plugins/filter/sanitize_spec.rb +++ b/spec/plugins/filter/sanitize_spec.rb @@ -35,10 +35,10 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items. + returned = subject.run + returned[0].items. count.should == 1 - subject.instance_variable_get(:@return_feeds)[0].items[0].description. + returned[0].items[0].description. should == 'fuga' } end @@ -64,10 +64,10 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items. + returned = subject.run + returned[0].items. count.should == 1 - subject.instance_variable_get(:@return_feeds)[0].items[0].description. + returned[0].items[0].description. should == 'fuga' } end @@ -93,10 +93,10 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items. + returned = subject.run + returned[0].items. count.should == 1 - subject.instance_variable_get(:@return_feeds)[0].items[0].description. + returned[0].items[0].description. should == 'fuga' } end @@ -122,10 +122,10 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items. + returned = subject.run + returned[0].items. count.should == 1 - subject.instance_variable_get(:@return_feeds)[0].items[0].description. + returned[0].items[0].description. should == 'fuga' } end @@ -147,10 +147,10 @@ its(:run) { should have(1).feeds } specify { - subject.run - subject.instance_variable_get(:@return_feeds)[0].items. + returned = subject.run + returned[0].items. count.should == 1 - subject.instance_variable_get(:@return_feeds)[0].items[0].description. + returned[0].items[0].description. should == '' } end diff --git a/spec/plugins/filter/tumblr_resize_spec.rb b/spec/plugins/filter/tumblr_resize_spec.rb index f997a27..61010a3 100644 --- a/spec/plugins/filter/tumblr_resize_spec.rb +++ b/spec/plugins/filter/tumblr_resize_spec.rb @@ -110,3 +110,32 @@ end end end + +# Tumblr has served images under two URL schemes. The size suffix above is the +# older one, which images uploaded before 2019 still carry; everything since +# carries the size as a path segment, and the plugin rewrites that as well. +describe Automatic::Plugin::FilterTumblrResize do + context "with the current media.tumblr.com URL scheme" do + subject { + Automatic::Plugin::FilterTumblrResize.new({}, + AutomaticSpec.generate_pipeline { + feed { + item "https://64.media.tumblr.com/aaa/bbb-1f/s540x810/ccc.jpg" + item "https://66.media.tumblr.com/aaa/bbb-1f/s75x75_c1/ccc.jpg" + item "https://64.media.tumblr.com/aaa/bbb-1f/s1280x1920/ccc.jpg" + item "https://example.com/not/a/tumblr/image.jpg" + }})} + + specify { + returned = subject.run + returned[0].items[0].link. + should == "https://64.media.tumblr.com/aaa/bbb-1f/s1280x1920/ccc.jpg" + returned[0].items[1].link. + should == "https://66.media.tumblr.com/aaa/bbb-1f/s75x75_c1/ccc.jpg" + returned[0].items[2].link. + should == "https://64.media.tumblr.com/aaa/bbb-1f/s1280x1920/ccc.jpg" + returned[0].items[3].link. + should == "https://example.com/not/a/tumblr/image.jpg" + } + end +end diff --git a/spec/plugins/notify/ikachan_spec.rb b/spec/plugins/notify/ikachan_spec.rb index bb03f55..8383543 100644 --- a/spec/plugins/notify/ikachan_spec.rb +++ b/spec/plugins/notify/ikachan_spec.rb @@ -1,58 +1,115 @@ +# -*- coding: utf-8 -*- +# Name:: Automatic::Plugin::Notify::Ikachan +# Author: id774 (More info: http://id774.net) +# Source Code:: https://github.com/id774/automaticruby +# License:: The GPL version 3, or LGPL version 3 (Dual License). +# Contact:: idnanashi@gmail.com +# Created:: Mar 9, 2012 +# Updated:: Aug 15, 2026 +# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. + require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') require 'notify/ikachan' +# NotifyIkachan needs an ikachan gateway, which is software the operator runs; +# doc/PLUGINS.md section 6.6 classifies it as Supported (external). What is +# verified here is the request the plugin builds, and no example reaches a +# gateway. describe Automatic::Plugin::NotifyIkachan do - paramz = { - "channels" => "#room", - "url" => "http://sample.com", - "port" => "4979", - "command" => "notice", + settings = { + 'channels' => 'room,#other', + 'url' => 'http://sample.com', + 'port' => '4979', + 'command' => 'notice', + 'interval' => 0 } + subject { Automatic::Plugin::NotifyIkachan.new( - paramz, + settings, AutomaticSpec.generate_pipeline { feed { item "http://github.com", "GitHub" } } ) } - it "should post title and link in the feed" do + it "posts the title and the link of each item, and returns the pipeline" do ikachan = double("ikachan") ikachan.should_receive(:post).with("http://github.com", "GitHub") - ikachan.should_receive(:params) subject.instance_variable_set(:@ikachan, ikachan) subject.run.should have(1).feed end + + it "adds a leading # to a channel that has none, and keeps one that has" do + subject.ikachan.params['channels'].should == ['#room', '#other'] + end end describe Automatic::Plugin::Ikachan do - describe "#post" do - subject { - Automatic::Plugin::Ikachan.new.tap {|ikachan| - ikachan.params = { - "channels" => "#room", - "url" => "http://sample.com", - "port" => "4979", - "command" => "notice", - } + subject { + Automatic::Plugin::Ikachan.new.tap { |ikachan| + ikachan.params = { + 'channels' => ['#room'], + 'url' => 'http://sample.com', + 'port' => '4979', + 'command' => 'notice' } } + } - specify { - link = "http://www.google.com" - - require 'net/http' - res = double("res") - res.should_receive(:code).and_return("200") - http = double("http") - http.should_receive(:post).with("/join", "channel=#room") - http.should_receive(:post).with( - "/notice", "channel=#room&message=#{link}").and_return(res) - http.should_receive(:start).and_yield(http) - proxy = Net::HTTP.stub(:new) { http } - subject.post(link) + describe "#post" do + let(:requests) { [] } + let(:http) { + response = double("response") + response.stub(:code).and_return("200") + double("http").tap { |connection| + connection.stub(:request) { |request| requests << request; response } + } } + + before do + Net::HTTP.stub(:Proxy).and_return( + double("proxy").tap { |proxy| proxy.stub(:start).and_yield(http) } + ) + end + + it "joins the channel and then posts the message, form encoded" do + subject.post("http://www.google.com") + + requests.map(&:path).should == ['/join', '/notice'] + requests[0].body.should == 'channel=%23room' + requests[1].body.should == 'channel=%23room&message=http%3A%2F%2Fwww.google.com' + end + + # An ampersand in a title used to split the request in two and lose the + # rest of the message. + it "encodes a title carrying a separator" do + subject.post("http://www.google.com", "Tom & Jerry") + + URI.decode_www_form(requests[1].body).to_h['message']. + should == 'Tom & Jerry - http://www.google.com' + end + + it "connects to the host and port the settings name" do + Net::HTTP.should_receive(:Proxy).with(nil, 8080).and_return( + double("proxy").tap { |proxy| + proxy.should_receive(:start). + with('sample.com', 4979, hash_including(use_ssl: false)).and_yield(http) + } + ) + subject.post("http://www.google.com") + end + + it "uses TLS where the gateway URL does" do + subject.params['url'] = 'https://sample.com' + Net::HTTP.should_receive(:Proxy).and_return( + double("proxy").tap { |proxy| + proxy.should_receive(:start). + with('sample.com', 4979, hash_including(use_ssl: true)).and_yield(http) + } + ) + subject.post("http://www.google.com") + end end end diff --git a/spec/plugins/provide/fluentd_spec.rb b/spec/plugins/provide/fluentd_spec.rb index c188121..0d865e4 100644 --- a/spec/plugins/provide/fluentd_spec.rb +++ b/spec/plugins/provide/fluentd_spec.rb @@ -5,52 +5,67 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jul 12, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# ProvideFluentd needs the optional fluent-logger gem; -# doc/PLUGINS.md section 6.5 classifies it as Supported (external). -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. +# ProvideFluentd needs the fluent-logger gem and a Fluentd instance; +# doc/PLUGINS.md section 6.5 classifies it as Supported (external). The gem is +# an optional dependency in its own Gemfile group, so this spec is skipped +# rather than stubbed where it is absent (doc/POLICY.md section 4). Where it is +# present, what is verified is what the plugin posts -- not that a Fluentd +# instance received it. return unless AutomaticSpec.plugin_available?('provide/fluentd') describe Automatic::Plugin::ProvideFluentd do - context 'when feed' do - describe 'should forward the feeds' do - hash = {} - hash['test1'] = "test2" - hash['test3'] = "test4" - expect = hash - - feeds = [] - json = hash.to_json - data = ActiveSupport::JSON.decode(json) - url = "http://id774.net/test/xml/data" - rss = Automatic::FeedMaker.content_provide(url, data) - feeds << rss - - subject { - Automatic::Plugin::ProvideFluentd.new( - { - 'host' => "localhost", - 'port' => "10000", - 'tag' => "automatic_spec.provide_fluentd", - 'mode' => "test" - }, - feeds - ) - } - - its (:run) { - fluentd = double("fluentd") - subject.run.should have(1).feed - subject.instance_variable_get(:@pipeline)[0].items[0].content_encoded.class == Hash - subject.instance_variable_get(:@pipeline)[0].items[0].content_encoded.should == expect - } + let(:settings) { + { 'host' => 'localhost', 'port' => '10000', 'tag' => 'automatic_spec.provide_fluentd' } + } + + let(:pipeline) { + [Automatic::FeedMaker.content_provide('http://id774.net/test/xml/data', + 'test1' => 'test2', 'test3' => 'test4')] + } + + context 'in test mode' do + subject { Automatic::Plugin::ProvideFluentd.new(settings.merge('mode' => 'test'), pipeline) } + + it 'builds no connection and returns the pipeline unchanged' do + Fluent::Logger::FluentLogger.should_not_receive(:open) + subject.run.should have(1).feed + end + end + + context 'with a logger' do + subject { Automatic::Plugin::ProvideFluentd.new(settings, pipeline) } + + before { + Fluent::Logger::FluentLogger.stub(:open).and_return(logger) + } + + let(:logger) { double('fluentd') } + + it "posts the item's content_encoded under the configured tag" do + logger.should_receive(:post). + with('automatic_spec.provide_fluentd', { 'test1' => 'test2', 'test3' => 'test4' }) + subject.run.should have(1).feed end + it 'opens the logger with the host and port from the settings' do + Fluent::Logger::FluentLogger.should_receive(:open). + with(nil, host: 'localhost', port: 10_000).and_return(logger) + logger.stub(:post) + subject.run + end + + # content_encoded has to be something Fluentd accepts as a record; a plain + # string is logged as an error and skipped rather than ending the run. + it 'logs a record the logger rejects' do + logger.stub(:post).and_raise(ArgumentError, 'not a hash') + Automatic::Log.stub(:puts) + Automatic::Log.should_receive(:puts).with('error', /not a hash/) + subject.run.should have(1).feed + end end end diff --git a/spec/plugins/publish/amazon_s3_spec.rb b/spec/plugins/publish/amazon_s3_spec.rb index 3e25e32..3543320 100644 --- a/spec/plugins/publish/amazon_s3_spec.rb +++ b/spec/plugins/publish/amazon_s3_spec.rb @@ -5,43 +5,94 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 25, 2014 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# PublishAmazonS3 is written against AWS SDK for Ruby v1; -# doc/PLUGINS.md section 6.7 classifies it as Needs rework. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('publish/amazon_s3') +require 'publish/amazon_s3' +require 'tmpdir' +# PublishAmazonS3 needs an S3 bucket and the aws-sdk-s3 gem, which is an +# optional dependency in its own Gemfile group; doc/PLUGINS.md section 6.7 +# classifies it as Supported (external). What is verified here is everything +# on this side of the bucket -- which items it selects, the key it computes, +# what `mode: test` does -- and none of it needs the gem, the account or the +# network. See doc/POLICY.md section 5. describe Automatic::Plugin::PublishAmazonS3 do - context 'when feed' do - describe 'should forward the feeds' do - subject { - Automatic::Plugin::PublishAmazonS3.new( - { - 'access_key' => "aabbcc", - 'secret_key' => "ddeeff", - 'bucket_name' => "test_bucket", - 'target_path' => "test/tmp", - 'mode' => "test" - }, - AutomaticSpec.generate_pipeline{ - feed { - item "http://github.com", "hoge", - "fuga" - } - } + let(:settings) { + { + 'access_key' => 'aabbcc', + 'secret_key' => 'ddeeff', + 'bucket_name' => 'test_bucket', + 'target_path' => 'test/tmp', + 'mode' => 'test' + } + } + + context 'with a link that is not a file URI' do + subject { + Automatic::Plugin::PublishAmazonS3.new( + settings, + AutomaticSpec.generate_pipeline { + feed { item 'http://github.com', 'hoge', 'fuga' } + } + ) + } + + it 'returns the pipeline unchanged and uploads nothing' do + subject.should_not_receive(:s3) + subject.run.should have(1).feed + end + end + + context 'with a file URI, in test mode' do + it 'names the key under target_path and does not build a client' do + Dir.mktmpdir do |dir| + path = File.join(dir, 'photo.png') + File.binwrite(path, 'x') + + plugin = Automatic::Plugin::PublishAmazonS3.new( + settings, + AutomaticSpec.generate_pipeline { feed { item "file://#{path}" } } ) - } + plugin.should_not_receive(:s3) + plugin.run.should have(1).feed + plugin.send(:target_key, path).should == 'test/tmp/photo.png' + end + end + end - its (:run) { - subject.run.should have(1).feed + describe 'the client settings' do + it 'passes the Recipe credentials through' do + plugin = Automatic::Plugin::PublishAmazonS3.new(settings.merge('region' => 'ap-northeast-1')) + plugin.send(:client_options).should == { + region: 'ap-northeast-1', + access_key_id: 'aabbcc', + secret_access_key: 'ddeeff' } end + # Without credentials in the Recipe the SDK's own chain is left to answer, + # which is how this runs from an instance role instead of from a secret in + # a file. + it 'passes nothing where the Recipe carries no credential' do + plugin = Automatic::Plugin::PublishAmazonS3.new('bucket_name' => 'b') + plugin.send(:client_options).should == {} + end + end + + describe 'an item whose file is gone' do + subject { + Automatic::Plugin::PublishAmazonS3.new( + settings.merge('mode' => 'live'), + AutomaticSpec.generate_pipeline { feed { item 'file:///nonexistent/photo.png' } } + ) + } + + it 'logs the failure and carries on' do + Automatic::Log.should_receive(:puts).with('error', /photo\.png/) + subject.run.should have(1).feed + end end end diff --git a/spec/plugins/publish/eject_spec.rb b/spec/plugins/publish/eject_spec.rb index be806b6..9f9011e 100644 --- a/spec/plugins/publish/eject_spec.rb +++ b/spec/plugins/publish/eject_spec.rb @@ -5,13 +5,17 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 9, 2013 -# Updated:: Jun 9, 2013 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') require 'publish/eject' +# PublishEject needs an optical drive and the command that drives it, which is +# the operator's machine rather than anything this suite can have. What is +# verified here is which command it would run and that the pipeline is +# returned unchanged; the drive itself is not opened. describe Automatic::Plugin::PublishEject do before do @pipeline = AutomaticSpec.generate_pipeline { @@ -19,24 +23,31 @@ end subject { - Automatic::Plugin::PublishEject.new({}, @pipeline) + Automatic::Plugin::PublishEject.new({ 'interval' => 0 }, @pipeline) } - it "should eject of feeds" do - subject.stub(:eject_cmd).and_return('echo') + it "returns the pipeline unchanged" do + subject.stub(:eject) subject.run.should have(1).items end - subject { - Automatic::Plugin::PublishEject.new({'interval' => 0}, @pipeline) - } + it "runs the open and the close command as argument vectors" do + subject.stub(:command_name).and_return('eject') + subject.should_receive(:system).with('eject').ordered + subject.should_receive(:system).with('eject', '-t').ordered + subject.run + end - it "should eject of feeds" do - subject.stub(:eject_cmd).and_return('echo') + it "says so rather than failing where no command is installed" do + subject.stub(:command_name).and_return(nil) + subject.should_not_receive(:system) + Automatic::Log.stub(:puts) + Automatic::Log.should_receive(:puts).with('warn', /No eject command/) subject.run.should have(1).items end - it "should eject_cmd" do - subject.eject_cmd.should_not == '' + it "looks the command up on PATH" do + subject.stub(:executable?) { |name| name == 'drutil' } + subject.send(:command_name).should == 'drutil' end end diff --git a/spec/plugins/publish/fluentd_spec.rb b/spec/plugins/publish/fluentd_spec.rb index 5fd6dbb..aa76569 100644 --- a/spec/plugins/publish/fluentd_spec.rb +++ b/spec/plugins/publish/fluentd_spec.rb @@ -5,43 +5,61 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 21, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# PublishFluentd needs the optional fluent-logger gem; -# doc/PLUGINS.md section 6.7 classifies it as Supported (external). -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. +# PublishFluentd needs the fluent-logger gem and a Fluentd instance; +# doc/PLUGINS.md section 6.7 classifies it as Supported (external). The gem is +# an optional dependency in its own Gemfile group, so this spec is skipped +# rather than stubbed where it is absent. See doc/POLICY.md section 4. return unless AutomaticSpec.plugin_available?('publish/fluentd') describe Automatic::Plugin::PublishFluentd do - context 'when feed' do - describe 'should forward the feeds' do - subject { - Automatic::Plugin::PublishFluentd.new( - { - 'host' => "localhost", - 'port' => "10000", - 'tag' => "automatic_spec.publish_fluent", - 'mode' => "test" - }, - AutomaticSpec.generate_pipeline{ - feed { - item "http://github.com", "hoge", - "fuga" - } - } - ) - } - - its (:run) { - fluentd = double("fluentd") - subject.run.should have(1).feed - } + let(:settings) { + { 'host' => 'localhost', 'port' => '10000', 'tag' => 'automatic_spec.publish_fluent' } + } + + let(:pipeline) { + AutomaticSpec.generate_pipeline { + feed { item 'http://github.com', 'hoge', 'fuga' } + } + } + + context 'in test mode' do + subject { Automatic::Plugin::PublishFluentd.new(settings.merge('mode' => 'test'), pipeline) } + + it 'builds no connection and returns the pipeline unchanged' do + Fluent::Logger::FluentLogger.should_not_receive(:open) + subject.run.should have(1).feed end + end + + context 'with a logger' do + subject { Automatic::Plugin::PublishFluentd.new(settings, pipeline) } + + let(:logger) { double('fluentd') } + before { Fluent::Logger::FluentLogger.stub(:open).and_return(logger) } + + it "posts the item's fields and a timestamp under the configured tag" do + posted = nil + logger.should_receive(:post) { |tag, record| tag.should == 'automatic_spec.publish_fluent' + posted = record } + subject.run.should have(1).feed + + posted[:title].should == 'hoge' + posted[:link].should == 'http://github.com' + posted[:description].should == 'fuga' + posted[:created_at].should match(%r{\A\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\z}) + end + + it 'logs a forward that fails and carries on' do + logger.stub(:post).and_raise(IOError, 'connection refused') + Automatic::Log.stub(:puts) + Automatic::Log.should_receive(:puts).with('warn', /connection refused/) + subject.run.should have(1).feed + end end end diff --git a/spec/plugins/publish/google_calendar_spec.rb b/spec/plugins/publish/google_calendar_spec.rb deleted file mode 100644 index 1d9b1d9..0000000 --- a/spec/plugins/publish/google_calendar_spec.rb +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::CustomFeed::SVNFLog -# Author: kzgs -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Feb 26, 2012 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -# The Google Calendar GData API v2 and ClientLogin were withdrawn by Google, and -# the gcalapi gem was last published in 2009; doc/PLUGINS.md section 6.7 -# classifies PublishGoogleCalendar as Unsupported. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('publish/google_calendar') -return unless AutomaticSpec.plugin_available?('gcalapi') - -describe Automatic::Plugin::PublishGoogleCalendar do - subject { - Automatic::Plugin::PublishGoogleCalendar.new( - {"username" => "user", "password" => "pswd"}, - AutomaticSpec.generate_pipeline{ - feed { - item "http://github.com", "GitHub" - } - } - ) - } - - it "should post the link in the feed" do - gc = double("gc") - gc.should_receive(:add).with("今日 GitHub") - subject.instance_variable_set(:@gc, gc) - subject.run.should have(1).feed - end -end - -describe Automatic::Plugin::Googlecalendar do - describe "#add" do - context "All day events" do - specify { - set_gcal_mock(all_day_event_mock( - "花火＠晴海", "", Time.mktime(2012, 8, 15))) - Automatic::Plugin::Googlecalendar.new.add("2012/8/15 花火＠晴海") - } - - specify { - lambda { - Automatic::Plugin::Googlecalendar.new.add("2012/2/30") - }.should raise_exception(RuntimeError, /不正な日付形式-1/) - } - end - end -end - -def all_day_event_mock(title, where, date=nil) - event = double("event") - { - :title => title, - :st => date.nil? ? nil : Time.mktime(date.year, date.month, date.day), - :en => nil, - :allday => true - }.each_pair do |key, value| - if value.nil? - event.should_receive("#{key}=".to_sym) - else - event.should_receive("#{key}=".to_sym).with(value) - end - end - event.should_receive(:st) - event.should_receive(:save!) - return event -end - -def cal_mock(event_mock) - cal = double("cal") - cal.should_receive(:create_event).and_return { - event_mock - } - return cal -end - -def set_gcal_mock(event_mock) - GoogleCalendar::Calendar.stub(:new) { - cal_mock(event_mock) - } -end diff --git a/spec/plugins/publish/hatena_bookmark_spec.rb b/spec/plugins/publish/hatena_bookmark_spec.rb index 8f96a67..9ebc44c 100644 --- a/spec/plugins/publish/hatena_bookmark_spec.rb +++ b/spec/plugins/publish/hatena_bookmark_spec.rb @@ -5,132 +5,99 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 22, 2012 -# Updated:: Feb 25, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') require 'publish/hatena_bookmark' +# PublishHatenaBookmark is classified Needs rework in doc/PLUGINS.md section +# 6.7: the service and its bookmarking API are current, the WSSE AtomPub +# interface this speaks is not. Nothing here asserts that a bookmark is made. +# What it holds is the part that a rework will have to keep working anyway -- +# how a link is made absolute, what the entry document contains -- and that +# the request goes over TLS, which is the defect that was worth correcting +# before the rest. describe Automatic::Plugin::PublishHatenaBookmark do - subject { + def plugin_for(link) Automatic::Plugin::PublishHatenaBookmark.new( - {"username" => "user", "password" => "pswd"}, - AutomaticSpec.generate_pipeline{ - feed { item "http://github.com" } - } - ) - } - - it "should post the link with prefix 'http' in the feed" do - hb = double("hb") - hb.should_receive(:post).with("http://github.com", nil) - subject.instance_variable_set(:@hb, hb) - subject.run.should have(1).feed - end -end - -describe Automatic::Plugin::PublishHatenaBookmark do - subject { - Automatic::Plugin::PublishHatenaBookmark.new( - {"username" => "user", "password" => "pswd"}, - AutomaticSpec.generate_pipeline{ - feed { item "//github.com" } - } - ) - } - - it "should post the link with prefix '//...' in the feed" do - hb = double("hb") - hb.should_receive(:post).with("http://github.com", nil) - subject.instance_variable_set(:@hb, hb) - subject.run.should have(1).feed - end -end - -describe Automatic::Plugin::PublishHatenaBookmark do - subject { - Automatic::Plugin::PublishHatenaBookmark.new( - {"username" => "user", "password" => "pswd"}, - AutomaticSpec.generate_pipeline{ - feed { item "https://github.com" } - } + { 'username' => 'user', 'password' => 'pswd', 'interval' => 0 }, + AutomaticSpec.generate_pipeline { feed { item link } } ) - } - - it "should post the link with prefix 'https' in the feed" do - hb = double("hb") - hb.should_receive(:post).with("https://github.com", nil) - subject.instance_variable_set(:@hb, hb) - subject.run.should have(1).feed end -end -describe Automatic::Plugin::PublishHatenaBookmark do - subject { - Automatic::Plugin::PublishHatenaBookmark.new( - {"username" => "user", "password" => "pswd"}, - AutomaticSpec.generate_pipeline{ - feed { item "github.com" } - } - ) - } - - it "should post the link with others in the feed" do - hb = double("hb") - hb.should_receive(:post).with("http://github.com", nil) - subject.instance_variable_set(:@hb, hb) - subject.run.should have(1).feed + { + 'http://github.com' => 'http://github.com', + 'https://github.com' => 'https://github.com', + '//github.com' => 'https://github.com', + 'github.com' => 'https://github.com' + }.each_pair do |given, expected| + it "posts #{given.inspect} as #{expected.inspect}" do + plugin = plugin_for(given) + hb = double('hb') + hb.should_receive(:post).with(expected, nil) + plugin.instance_variable_set(:@hb, hb) + plugin.run.should have(1).feed + end end end describe Automatic::Plugin::HatenaBookmark do - describe "#wsse" do - subject { - Automatic::Plugin::HatenaBookmark.new.wsse("anonymous", "pswd") - } - - it { should be_has_key('X-WSSE') } - - specify { - subject['X-WSSE'].should match( - /^UsernameToken\sUsername="anonymous",\sPasswordDigest=".+", Nonce=".+", Created="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z"/) - } + subject { Automatic::Plugin::HatenaBookmark.new } + + describe '#wsse' do + it 'builds the header' do + header = subject.wsse('anonymous', 'pswd') + header.should be_has_key('X-WSSE') + header['X-WSSE'].should match( + /\AUsernameToken Username="anonymous", PasswordDigest=".+", Nonce=".+", Created="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z"\z/ + ) + end + + it 'uses a different nonce every time' do + nonce = ->(header) { header['X-WSSE'][/Nonce="([^"]+)"/, 1] } + nonce.call(subject.wsse('anonymous', 'pswd')). + should_not == nonce.call(subject.wsse('anonymous', 'pswd')) + end end - describe "#post" do - subject { - Automatic::Plugin::HatenaBookmark.new - } - - specify { - url = "http://www.google.com" - comment = "Can we trust them ?" + describe '#post' do + let(:requests) { [] } - require 'net/http' - res = double("res") - res.should_receive(:code).and_return("201") - http = double("http") - http.should_receive(:post).with("/atom/post", subject.toXml(url, comment), - subject.wsse("", "")).and_return(res) - http.should_receive(:start).and_yield(http) - proxy = Net::HTTP.stub(:new) { http } - subject.post(url, comment) - } - - specify { - url = "http://www.google.com" - comment = "Can we trust them ?" - - require 'net/http' - res = double("res") - res.should_receive(:code).twice.and_return("400") - http = double("http") - http.should_receive(:post).with("/atom/post", subject.toXml(url, comment), - subject.wsse("", "")).and_return(res) - http.should_receive(:start).and_yield(http) - proxy = Net::HTTP.stub(:new) { http } - subject.post(url, comment) - } + def connection(code) + response = double('response') + response.stub(:code).and_return(code) + double('http').tap { |http| + http.stub(:request) { |request| requests << request; response } + } + end + + it 'posts the entry to the endpoint over TLS' do + Net::HTTP.should_receive(:Proxy).with(nil, 8080).and_return( + double('proxy').tap { |proxy| + proxy.should_receive(:start). + with('b.hatena.ne.jp', 443, hash_including(use_ssl: true)). + and_yield(connection('201')) + } + ) + + subject.post('http://www.google.com', 'Can we trust them ?') + + requests[0].path.should == '/atom/post' + requests[0]['x-wsse'].should_not be_nil + requests[0].body.should include('href="http://www.google.com"') + requests[0].body.should include('Can we trust them ?') + end + + it 'logs any other response code as an error' do + Net::HTTP.stub(:Proxy).and_return( + double('proxy').tap { |proxy| proxy.stub(:start).and_yield(connection('400')) } + ) + Automatic::Log.stub(:puts) + Automatic::Log.should_receive(:puts).with(:error, /400 Error/) + + subject.post('http://www.google.com', nil) + end end end diff --git a/spec/plugins/publish/hipchat_spec.rb b/spec/plugins/publish/hipchat_spec.rb deleted file mode 100644 index ad86bd4..0000000 --- a/spec/plugins/publish/hipchat_spec.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Hipchat -# Author: Kohei Hasegawa (More info: http://github.com/banyan) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Jun 5, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# HipChat was shut down by Atlassian; -# doc/PLUGINS.md section 6.7 classifies PublishHipchat as Unsupported. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('publish/hipchat') - -describe Automatic::Plugin::PublishHipchat do - let(:config) { - { - 'api_token' => "bogus_api_token", - 'room_id' => 'bogus_room', - 'username' => 'bogus_bot', - 'interval' => 1, - 'retry' => 1 - } - } - - let(:pipeline) { - AutomaticSpec.generate_pipeline { - feed { item("http://github.com", 'title', 'description') } - } - } - - context 'return feed' do - subject { - described_class.new(config, pipeline) - } - - context 'when successfully' do - it "should passed proper argument to HipChat::Client" do - client = double('client').as_null_object - HipChat::Client.should_receive(:new).with("bogus_api_token").and_return(client) - subject.run - end - - it "should post the link in the feed" do - client = double("client") - client.should_receive(:send).with('bogus_bot', 'description', {"color"=>"yellow", "notify"=>false}) - subject.instance_variable_set(:@client, client) - subject.run.should have(1).feed - end - end - - context 'when raise an error during post' do - it do - client = double("client") - client.stub(:send).and_raise - subject.instance_variable_set(:@client, client) - Automatic::Log.should_receive(:puts).twice - subject.run.should have(1).feed - end - end - end - - context 'when feed is empty' do - subject { - described_class.new(config) - } - - it "should not post" do - subject.run.should have(0).feed - end - end -end diff --git a/spec/plugins/publish/instapaper_spec.rb b/spec/plugins/publish/instapaper_spec.rb index a55b264..4d2ca00 100644 --- a/spec/plugins/publish/instapaper_spec.rb +++ b/spec/plugins/publish/instapaper_spec.rb @@ -5,79 +5,91 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 9, 2013 -# Updated:: Feb 25, 2014 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') require 'publish/instapaper' +# PublishInstapaper needs an Instapaper account; doc/PLUGINS.md section 6.7 +# classifies it as Supported (external). Everything up to the request is +# verified here -- what is posted where, with which authentication, and what +# each response code means -- and no example reaches the service. The +# constructor authenticates, so every example stubs the request that does it. describe Automatic::Plugin::PublishInstapaper do - context 'when feed' do - subject { - Automatic::Plugin::PublishInstapaper.new( - { 'email' => "email@example.com", - 'password' => "pswd", - 'interval' => 5, - 'retry' => 5 - }, - AutomaticSpec.generate_pipeline { - feed { item "http://github.com" } - })} + let(:settings) { + { 'email' => 'email@example.com', 'password' => 'pswd', + 'interval' => 0, 'retry' => 1 } + } - it "should post the link in the feed" do - instapaper = double("instapaper") - instapaper.should_receive(:add).with("http://github.com", nil, '') - subject.instance_variable_set(:@instapaper, instapaper) - subject.run.should have(1).feed - end - end + before { Automatic::Plugin::Instapaper.any_instance.stub(:request) } - context 'when feed is empty' do - subject { - Automatic::Plugin::PublishInstapaper.new( - { 'email' => "email@example.com", - 'password' => "pswd", - 'interval' => 1, - 'retry' => 1 - }, - AutomaticSpec.generate_pipeline { - feed { item "http://github.com" } - })} + subject { + Automatic::Plugin::PublishInstapaper.new( + settings, + AutomaticSpec.generate_pipeline { feed { item 'http://github.com' } } + ) + } - its (:run) { subject.run.should have(1).feed } + it 'adds the link in the feed and returns the pipeline' do + instapaper = double('instapaper') + instapaper.should_receive(:add).with('http://github.com', nil, '') + subject.instance_variable_set(:@instapaper, instapaper) + subject.run.should have(1).feed + end + + it 'retries a failing item and carries on' do + instapaper = double('instapaper') + instapaper.should_receive(:add).twice.and_raise( + Automatic::Plugin::Instapaper::Error, 'Instapaper answered 403' + ) + subject.instance_variable_set(:@instapaper, instapaper) + subject.run.should have(1).feed end end describe Automatic::Plugin::Instapaper do - describe "#add" do - subject { - Automatic::Plugin::Instapaper.new( - { 'email' => "email@example.com", - 'password' => "pswd", - 'interval' => 5, - 'retry' => 5 - })} + before { Automatic::Plugin::Instapaper.any_instance.stub(:request) } + + subject { Automatic::Plugin::Instapaper.new('email@example.com', 'pswd') } + + describe '#add' do + it 'returns the response when the service accepts the URL' do + response = double('response', code: '201') + subject.should_receive(:request). + with(:add, url: 'http://www.google.com', title: 'a title', selection: 'a body'). + and_return(response) + subject.add('http://www.google.com', 'a title', 'a body').should == response + end + + it 'raises on any other response code' do + subject.stub(:request).and_return(double('response', code: '403')) + lambda { + subject.add('http://www.google.com', 'a title', 'a body') + }.should raise_error(Automatic::Plugin::Instapaper::Error, /403/) + end + end - url = "http://www.google.com" - title = "automatic test" - description = "automatic test" + describe 'the request it builds' do + # Unstubbed here, because the request itself is what is being checked; + # Net::HTTP.start is stubbed, so nothing is sent. + before { Automatic::Plugin::Instapaper.any_instance.unstub(:request) } - specify { - res = double("res") - res.should_receive(:code).and_return("201") - subject.should_receive(:request).and_return(res) - subject.add(url, title, description) - } + it 'posts the form to the Simple API over TLS, with the certificate verified' do + posted = nil + Net::HTTP.should_receive(:start). + with('www.instapaper.com', 443, + hash_including(use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_PEER)) { |*_args, &block| + http = double('http') + http.stub(:request) { |request| posted = request; double('response', code: '201') } + block.call(http) + }.at_least(:once) + Automatic::Plugin::Instapaper.new('email@example.com', 'pswd') - it 'raise error' do - lambda{ - res = double("res") - res.should_receive(:code).twice.and_return("403") - subject.should_receive(:request).and_return(res) - subject.add(url, title, description) - }.should raise_error + posted.path.should == '/api/authenticate' + posted['authorization'].should == "Basic #{['email@example.com:pswd'].pack('m0')}" end end end diff --git a/spec/plugins/publish/memcached_spec.rb b/spec/plugins/publish/memcached_spec.rb index d6ae342..16076ac 100644 --- a/spec/plugins/publish/memcached_spec.rb +++ b/spec/plugins/publish/memcached_spec.rb @@ -5,66 +5,63 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Jun 25, 2013 -# Updated:: Aug 14, 2026 +# Updated:: Aug 15, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') -# PublishMemcached needs the optional dalli gem; -# doc/PLUGINS.md section 6.7 classifies it as Supported (external). -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. +# PublishMemcached needs the dalli gem and a memcached server; doc/PLUGINS.md +# section 6.7 classifies it as Supported (external). The gem is an optional +# dependency in its own Gemfile group, so this spec is skipped rather than +# stubbed where it is absent (doc/POLICY.md section 4). Where it is present, +# what is verified is the value the plugin builds and the key it stores it +# under -- no server is contacted. return unless AutomaticSpec.plugin_available?('publish/memcached') describe Automatic::Plugin::PublishMemcached do - context 'when feed' do - describe 'should put the feeds to memcached' do - subject { - Automatic::Plugin::PublishMemcached.new( - { - 'host' => "localhost", - 'port' => "11211", - 'key' => "rspec" - }, - AutomaticSpec.generate_pipeline{ - feed { - item "http://blog.id774.net/post/2012/01/30/18/", "ブログをはじめた", - "なぜいまブログなのか - - いままでインターネット全体に公開するブログとして、はてなダイアリーを利用してきた。それ以外のある程度まとまった文章は Facebook に書いてきた。それはそれで良かったのだけど、いろいろと思うところもあり、このたび新しくブログをはじめることにした。 - - はてなダイアリーはシンタックスハイライト (プログラミング言語の色付けのこと) が充実していたので利用していた。しかし最近登場した CoffeeScript や Haml のような新しい言語には対応していない。新しくはてなブログというのも始まったが、ダイアリー以上にシンタックスハイライトが使えないようだ。 - - そこで、今後インターネット上で文章を書いていくにあたりどうするか考えた。" - item "http://blog.id774.net/post/2012/01/30/38/", "Twitter Viewer つくった", - "Twitter を閲覧するための Web アプリをつくった。 - - Twitter Viewer - - やっていることは至ってシンプルで RDB にためた発言をブラウザに表示させているだけである。内容は Rails の Scaffold ほとんどそのまま。 CSS はサイトローカルな Bootstrap を読み込んでいる。簡単なアプリだが、ブラウザにいちど表示させてしまえば電波が入らない地下鉄などでもゆっくり読めるので、モバイル環境で大量の発言をざっとチェックしたいときなどに使えて意外と実用的である。発言のクロールは別途おこなう必要がある。この例では Termtter の ActiveRecord プラグインを利用している。" - item "http://blog.id774.net/post/2012/01/30/48/", "PC-98 とエミュレータ", - "Facebook には少し書いたのだが、今年に入ってから 90 年代に使っていた PC-98 と呼ばれる PC を発掘したので起動した。もう 15 年前後も経っているというのに正常に利用することができて感動してしまった。あの ThinkPad ですら数年ほど電源を入れないで放置しておくと起動しないことが多いのに、さすが発売当初 40 〜 50 万円程もした高級マシンである。そんなわけで今回は PC-98 の話。 - - PC-98 のソフトを使う - 当時のソフトウェアを利用するためには以下のものが必要だ。 - 1. PC-98 エミュレータ - 2. MS-DOS (オペレーティングシステム) - 3. 動作させる対象のソフトウェア" - } - feed { - item "http://d.hatena.ne.jp/Naruhodius/20120130/1327862031", "ブログを移転しました", - "いままでこの「はてなダイアリー」にブログを書いてきましたが、以下のアドレスにブログを移転することにしました。このブログはもう更新されません。以下の新しいブログを購読してください。" - } - } - ) - } + let(:cache) { double('cache') } - its (:run) { - fluentd = double("memcached") - subject.run.should have(2).feed + let(:pipeline) { + AutomaticSpec.generate_pipeline { + feed { + item 'http://blog.id774.net/post/2012/01/30/18/', 'ブログをはじめた', 'なぜいまブログなのか' + item 'http://blog.id774.net/post/2012/01/30/38/', 'Twitter Viewer つくった', '本文' + } + feed { + item 'http://d.hatena.ne.jp/Naruhodius/20120130/1327862031', 'ブログを移転しました', '本文' } - end + } + } + + subject { + Automatic::Plugin::PublishMemcached.new( + { 'host' => 'localhost', 'port' => 11_211, 'key' => 'rspec' }, pipeline + ) + } + + before { Dalli::Client.stub(:new).and_return(cache) } + + it 'stores the whole pipeline under one key, keyed by link' do + stored = nil + cache.should_receive(:set) { |key, value| key.should == 'rspec'; stored = value } + subject.run.should have(2).feeds + + stored.keys.should have(3).links + stored['http://blog.id774.net/post/2012/01/30/18/'][:title].should == 'ブログをはじめた' + end + + # `port: 11211` in a Recipe is an Integer, and building the server address by + # concatenation used to end the run on it. + it 'accepts a port written as a number or as a string' do + Dalli::Client.should_receive(:new).with('localhost:11211').and_return(cache) + cache.stub(:set) + subject.run + end + it 'logs a failure to store and returns the pipeline' do + cache.stub(:set).and_raise(RuntimeError, 'connection refused') + Automatic::Log.stub(:puts) + Automatic::Log.should_receive(:puts).with('warn', /connection refused/) + subject.run.should have(2).feeds end end diff --git a/spec/plugins/publish/pocket_spec.rb b/spec/plugins/publish/pocket_spec.rb deleted file mode 100644 index b978632..0000000 --- a/spec/plugins/publish/pocket_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Pocket -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 15, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -# Pocket was shut down in July 2025; -# doc/PLUGINS.md section 6.7 classifies PublishPocket as Unsupported. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('publish/pocket') - -describe Automatic::Plugin::PublishPocket do - context 'return feed' do - subject { - Automatic::Plugin::PublishPocket.new( - { 'consumer_key' => "hugehuge", - 'access_token' => "hogehoge", - 'interval' => 1, - 'retry' => 1 - }, - AutomaticSpec.generate_pipeline { - feed { item "http://github.com" } - })} - - it "should post the link in the feed" do - client = double("client") - client.should_receive(:add).with(:url => 'http://github.com') - subject.instance_variable_set(:@client, client) - subject.run.should have(1).feed - end - - it "should not post" do - subject.run.should have(1).feed - end - end - - context 'not return feed' do - subject { - Automatic::Plugin::PublishPocket.new( - { 'consumer_key' => "hugehuge", - 'access_token' => "hogehoge", - 'interval' => 1, - 'retry' => 1 - })} - - it "should un post" do - subject.run.should have(0).feed - end - end -end diff --git a/spec/plugins/publish/twitter_spec.rb b/spec/plugins/publish/twitter_spec.rb deleted file mode 100644 index cd78d89..0000000 --- a/spec/plugins/publish/twitter_spec.rb +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Publish::Twitter -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 5, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -# PublishTwitter uses the twitter gem's v4 interface, which no longer exists; -# doc/PLUGINS.md section 6.7 classifies it as Unsupported. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('publish/twitter') - -describe Automatic::Plugin::PublishTwitter do - context 'when feed' do - describe 'should post the link tweet' do - subject { - Automatic::Plugin::PublishTwitter.new( - {}, - AutomaticSpec.generate_pipeline{ - feed { item "http://github.com" } - })} - - its (:run) { - twitter = double("twitter") - twitter.should_receive(:update).with(" http://github.com") - subject.instance_variable_set(:@twitter, twitter) - subject.run.should have(1).feed - } - end - - describe 'should post the tweet_tmp' do - subject { - Automatic::Plugin::PublishTwitter.new( - { 'tweet_tmp' => 'publish-twitter'}, - AutomaticSpec.generate_pipeline{ - feed { item "http://github.com" } - })} - - its (:run) { - twitter = double("twitter") - twitter.should_receive(:update).with("publish-twitter") - subject.instance_variable_set(:@twitter, twitter) - subject.run.should have(1).feed - } - end - - describe 'interval & retry was used error' do - subject { - Automatic::Plugin::PublishTwitter.new( - { 'interval' => 1, 'retry' => 1 }, - AutomaticSpec.generate_pipeline{ - feed { item "http://github.com" } - })} - - its (:run) { - subject.run.should have(1).feed - } - end - end - - context 'when feed is empty' do - describe 'should not post' do - subject { - Automatic::Plugin::PublishTwitter.new( - {}, - )} - - its (:run) { - subject.run.should have(0).feed - } - end - end -end diff --git a/spec/plugins/store/file_spec.rb b/spec/plugins/store/file_spec.rb index 250e160..51b5859 100644 --- a/spec/plugins/store/file_spec.rb +++ b/spec/plugins/store/file_spec.rb @@ -46,7 +46,7 @@ { "path" => dir, 'retry' => 1, - 'interval' => 2 + 'interval' => 0 }, AutomaticSpec.generate_pipeline { feed { item "aaa" } @@ -57,3 +57,74 @@ end end + +describe Automatic::Plugin::StoreFile do + # A link arrives from a feed, which is to say from outside; a store plugin + # that would read file:// on being asked to is a store plugin that can be + # asked to read anything. + it "refuses a link that is not HTTP or HTTPS" do + Dir.mktmpdir do |dir| + instance = Automatic::Plugin::StoreFile.new( + { "path" => dir }, + AutomaticSpec.generate_pipeline { feed { item "file:///etc/passwd" } } + ) + instance.run.should have(0).feed + Dir.children(dir).should be_empty + end + end + + it "writes what it fetched and rewrites the link to a file URI" do + Dir.mktmpdir do |dir| + Automatic::Http.stub(:read).and_return('a body') + instance = Automatic::Plugin::StoreFile.new( + { "path" => dir }, + AutomaticSpec.generate_pipeline { feed { item "https://example.com/a/photo.png" } } + ) + + returned = instance.run + returned.should have(1).feed + returned[0].items[0].link.should == "file://#{File.join(dir, 'photo.png')}" + File.read(File.join(dir, 'photo.png')).should == 'a body' + end + end + + # `s3n` is what Recipes written for this plugin use; `s3` is the spelling + # everything else uses and is accepted as well. Both go to the SDK rather + # than over HTTP. + %w[s3 s3n].each do |scheme| + it "fetches a #{scheme}:// link from the bucket" do + Dir.mktmpdir do |dir| + client = double('s3') + instance = Automatic::Plugin::StoreFile.new( + { "path" => dir, "bucket_name" => "a-bucket" }, + AutomaticSpec.generate_pipeline { feed { item "#{scheme}://ignored/a/photo.png" } } + ) + instance.stub(:s3).and_return(client) + client.should_receive(:get_object). + with(bucket: 'a-bucket', key: 'a/photo.png', + response_target: File.join(dir, 'photo.png')) + + instance.run[0].items[0].link.should == "file://#{File.join(dir, 'photo.png')}" + end + end + end + + describe "the client settings" do + subject { + Automatic::Plugin::StoreFile.new( + 'path' => '/tmp', 'region' => 'ap-northeast-1', + 'access_key' => 'aabbcc', 'secret_key' => 'ddeeff' + ) + } + + it "passes the Recipe credentials through" do + subject.send(:client_options).should == { + region: 'ap-northeast-1', access_key_id: 'aabbcc', secret_access_key: 'ddeeff' + } + end + + it "passes nothing where the Recipe carries no credential" do + Automatic::Plugin::StoreFile.new('path' => '/tmp').send(:client_options).should == {} + end + end +end diff --git a/spec/plugins/subscription/chan_toru_spec.rb b/spec/plugins/subscription/chan_toru_spec.rb deleted file mode 100644 index 377ca8b..0000000 --- a/spec/plugins/subscription/chan_toru_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::ChanToru -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Jun 28, 2013 -# Updated:: Jun 28, 2013 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -require 'subscription/chan_toru' - -def chan_toru(config = {}, pipeline = []) - Automatic::Plugin::SubscriptionChanToru.new(config,pipeline) -end - -describe 'Automatic::Plugin::SubscriptionChanToru' do - context 'when feed is empty' do - describe 'attestation error' do - subject { chan_toru } - before do - subject.should_receive(:g_guide_pipeline).times.and_return([]) - end - its(:run) { should be_empty } - end - - describe 'interval & retry was used error' do - config = {'interval' => 1, 'retry' => 1} - subject { chan_toru(config) } - before do - subject. - should_receive(:g_guide_pipeline). - exactly(2). - times.and_return('') - end - its(:run) { should be_empty } - end - end - - context 'when feed' do - describe 'config keyword' do - subject { chan_toru } - before do - pipeline = AutomaticSpec.generate_pipeline { - feed { - item "http://soramugi.net/images/hugehuge" - item "http://tv.so-net.ne.jp/schedule/500333201307052330.action?from=rss" - }} - subject. - should_receive(:g_guide_pipeline). - times.and_return(pipeline) - end - its(:run) { should have(1).feed } - end - end -end diff --git a/spec/plugins/subscription/feed_spec.rb b/spec/plugins/subscription/feed_spec.rb index d0d7fdc..3e6b902 100644 --- a/spec/plugins/subscription/feed_spec.rb +++ b/spec/plugins/subscription/feed_spec.rb @@ -64,11 +64,33 @@ Automatic::Plugin::SubscriptionFeed.new( { 'feeds' => ["invalid_url"], 'retry' => 1, - 'interval' => 1 + 'interval' => 0 } ) } its(:run) { should be_empty } end + + # `interval` is seconds between attempts, and it was not being waited: the + # line meant to do it assigned to a local variable named sleep and returned + # at once, so a Recipe asking to be gentle with a host was not. + context "with an interval between attempts" do + subject { + Automatic::Plugin::SubscriptionFeed.new( + { 'feeds' => ["invalid_url"], 'retry' => 2, 'interval' => 7 } + ) + } + + it "waits between them" do + subject.should_receive(:sleep).with(7).twice + subject.run.should be_empty + end + end + + context "with no feeds at all" do + subject { Automatic::Plugin::SubscriptionFeed.new(nil) } + + its(:run) { should be_empty } + end end diff --git a/spec/plugins/subscription/g_guide_spec.rb b/spec/plugins/subscription/g_guide_spec.rb deleted file mode 100644 index 5add3cd..0000000 --- a/spec/plugins/subscription/g_guide_spec.rb +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::GGuide -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Jun 28, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -require 'subscription/g_guide' - -def g_guide(config = {}, pipeline = []) - Automatic::Plugin::SubscriptionGGuide.new(config,pipeline) -end - -describe 'Automatic::Plugin::SubscriptionGGuide' do - context 'when config' do - describe 'is empty' do - subject { g_guide } - - its(:feed_url) { - should == Automatic::Plugin::SubscriptionGGuide::G_GUIDE_RSS + - 'stationPlatformId=0&' - } - end - describe 'keyword is anime' do - config = { 'keyword' => 'anime' } - subject { g_guide(config) } - - it 'feed_url' do - subject.feed_url(config['keyword']).should == URI::RFC2396_Parser.new.escape( - Automatic::Plugin::SubscriptionGGuide::G_GUIDE_RSS + - "condition.keyword=#{config['keyword']}&" + - 'stationPlatformId=0&') - end - end - describe 'station is 地上波' do - config = { 'station' => '地上波' } - subject { g_guide(config) } - - its(:feed_url) { - should == URI::RFC2396_Parser.new.escape( - Automatic::Plugin::SubscriptionGGuide::G_GUIDE_RSS + - 'stationPlatformId=1&') - } - end - end - - context 'when feed is empty' do - describe 'attestation error' do - subject { g_guide } - before do - subject.should_receive(:feed_url).and_return('') - end - its(:run) { should be_empty } - end - - describe 'interval & retry was used error' do - config = {'interval' => 1, 'retry' => 1} - subject { g_guide(config) } - before do - subject.should_receive(:feed_url).exactly(2).times.and_return('') - end - its(:run) { should be_empty } - end - end - - context 'when feed', :network do - describe 'config keyword' do - config = { 'keyword' => 'アニメ', 'station' => '地上波' } - subject { g_guide(config) } - its(:run) { should have(1).feed } - end - - describe 'config keyword ","' do - config = { 'keyword' => 'おじゃる丸,忍たま', 'station' => '地上波' } - subject { g_guide(config) } - its(:run) { should have(2).feed } - end - end -end diff --git a/spec/plugins/subscription/link_spec.rb b/spec/plugins/subscription/link_spec.rb index 53b8beb..d2dc5f1 100644 --- a/spec/plugins/subscription/link_spec.rb +++ b/spec/plugins/subscription/link_spec.rb @@ -63,7 +63,7 @@ subject { Automatic::Plugin::SubscriptionLink.new( { 'urls' => ["invalid_url"], - 'interval' => 1, + 'interval' => 0, 'retry' => 1 } ) diff --git a/spec/plugins/subscription/pocket_spec.rb b/spec/plugins/subscription/pocket_spec.rb deleted file mode 100644 index 40b9be2..0000000 --- a/spec/plugins/subscription/pocket_spec.rb +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::Pocket -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 21, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -# Pocket was shut down in July 2025; -# doc/PLUGINS.md section 6.1 classifies SubscriptionPocket as Unsupported. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('subscription/pocket') - -def pocket(config = {}, pipeline = []) - Automatic::Plugin::SubscriptionPocket.new(config,pipeline) -end - -describe 'Automatic::Plugin::SubscriptionPocket' do - context 'when feed is empty' do - describe 'attestation error' do - subject { pocket } - - its(:run) { should be_empty } - end - - describe 'interval & retry was used error' do - config = {'interval' => 1, 'retry' => 1} - subject { pocket(config) } - - its(:run) { should be_empty } - end - end - - context 'when feed' do - describe 'config optional' do - config = { 'optional' => { - 'count' => 1, - 'favorite' => 1 - }} - subject { pocket(config) } - before do - retrieve = {'list' => { - 'id' => { - 'given_url' => 'http://github.com', - 'given_title' => 'GitHub', - 'excerpt' => 'github' - }}} - client = double("client") - client.should_receive(:retrieve). - with(config['optional']). - and_return(retrieve) - subject.instance_variable_set(:@client, client) - end - - its(:run) { should have(1).item } - end - end -end diff --git a/spec/plugins/subscription/tumblr_spec.rb b/spec/plugins/subscription/tumblr_spec.rb index 69d30c9..57826af 100644 --- a/spec/plugins/subscription/tumblr_spec.rb +++ b/spec/plugins/subscription/tumblr_spec.rb @@ -65,7 +65,7 @@ Automatic::Plugin::SubscriptionTumblr.new( { 'urls' => ["invalid_url"], 'pages' => 3, - 'interval' => 1, + 'interval' => 0, 'retry' => 2 } ) @@ -74,3 +74,50 @@ its(:run) { should be_empty } end end + +# The examples below read a page rather than reach one, so they run without a +# network -- but the reading is FeedParser.parse_html, which needs nokogiri. +# That gem is in the Gemfile's optional :plugins group, which the default +# suite does not install. See doc/POLICY.md section 5. +if AutomaticSpec.optional_dependency?('nokogiri') +describe Automatic::Plugin::SubscriptionTumblr do + let(:page) { + 'one' \ + 'two' + } + + before { Automatic::Http.stub(:read).and_return(page) } + + # A theme's page carries the blog's own posts and a great deal else. A link + # that leaves the blog's host is blanked rather than removed, which is the + # pipeline's way of saying "not applicable". + it "blanks the links that leave the blog's host" do + plugin = Automatic::Plugin::SubscriptionTumblr.new( + 'urls' => ['http://example.tumblr.com'], 'interval' => 0 + ) + links = plugin.run[0].items.map(&:link) + links.compact.should == ['http://example.tumblr.com/post/1'] + links.should have(2).links + end + + it "walks back through the pages the settings ask for" do + Automatic::Http.should_receive(:read). + with('http://example.tumblr.com').ordered.and_return(page) + Automatic::Http.should_receive(:read). + with('http://example.tumblr.com/page/2').ordered.and_return(page) + Automatic::Http.should_receive(:read). + with('http://example.tumblr.com/page/3').ordered.and_return(page) + + Automatic::Plugin::SubscriptionTumblr.new( + 'urls' => ['http://example.tumblr.com'], 'pages' => 3, 'interval' => 0 + ).run.should have(3).feeds + end + + it "fetches the blog's own page only where no page count is given" do + Automatic::Http.should_receive(:read).once.and_return(page) + Automatic::Plugin::SubscriptionTumblr.new( + 'urls' => ['http://example.tumblr.com'], 'interval' => 0 + ).run.should have(1).feed + end +end +end diff --git a/spec/plugins/subscription/twitter_search_spec.rb b/spec/plugins/subscription/twitter_search_spec.rb deleted file mode 100644 index 8cf1e5b..0000000 --- a/spec/plugins/subscription/twitter_search_spec.rb +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::TwitterSearch -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 30, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -# SubscriptionTwitterSearch uses the twitter gem's v4 interface, which no longer -# exists; doc/PLUGINS.md section 6.1 classifies it as Unsupported. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('subscription/twitter_search') - -def twitter_search(config = {}, pipeline = []) - Automatic::Plugin::SubscriptionTwitterSearch.new(config,pipeline) -end - -describe 'Automatic::Plugin::SubscriptionTwitterSearch' do - context 'when feed is empty' do - - describe 'attestation error' do - subject { twitter_search } - - its(:run) { should be_empty } - end - - describe 'interval & retry was used error' do - config = {'interval' => 1, 'retry' => 1} - subject { twitter_search(config) } - - its(:run) { should be_empty } - end - - end - - context 'when feed' do - describe 'config optional' do - config = { 'search' => 'ruby', 'opt' => { 'lang' => 'ja', 'count' => 1 }} - subject { twitter_search(config) } - before do - status = Hashie::Mash.new - status.user = {'screen_name' => 'soramugi'} - status.id = 12345 - status.text = 'twitter_search rspec' - status.created_at = Time.now - search = Hashie::Mash.new - search.results = [status] - client = double("client") - client.should_receive(:search) - .with(config['search'],config['opt']) - .and_return(search) - subject.instance_variable_set(:@client, client) - end - - its(:run) { should have(1).item } - end - end -end diff --git a/spec/plugins/subscription/twitter_spec.rb b/spec/plugins/subscription/twitter_spec.rb deleted file mode 100644 index 85b36d7..0000000 --- a/spec/plugins/subscription/twitter_spec.rb +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::Twitter -# Author: id774 (More info: http://id774.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: Sep 10, 2012 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -# SubscriptionTwitter reads HTML with nokogiri, which the Gemfile declares in -# its optional :plugins group. The default suite and CI do not install it, so -# this spec runs only where the operator has. See doc/POLICY.md section 5. -return unless AutomaticSpec.optional_dependency?('nokogiri') - -require 'subscription/twitter' - -describe Automatic::Plugin::SubscriptionTwitter do - context "with empty URLs" do - subject { - Automatic::Plugin::SubscriptionTwitter.new( - { 'urls' => [] } - ) - } - - its(:run) { should be_empty } - end - - context "with URLs whose invalid URL" do - subject { - Automatic::Plugin::SubscriptionTwitter.new( - { 'urls' => ["invalid_url"] } - ) - } - - its(:run) { should be_empty } - end - - context "with URLs whose valid URL", :network do - subject { - Automatic::Plugin::SubscriptionTwitter.new( - { 'urls' => [ - "http://id774.net/test/twitter/favorites.html"], - 'interval' => 1 - } - ) - } - - its(:run) { should have(1).item } - end - - context "with retry to URLs whose valid URL", :network do - subject { - Automatic::Plugin::SubscriptionTwitter.new( - { 'urls' => [ - "http://id774.net/test/twitter/favorites.html"], - 'interval' => 2, - 'retry' => 1 - } - ) - } - - its(:run) { should have(1).item } - end - - context "with retry to URLs whose invalid URL" do - subject { - Automatic::Plugin::SubscriptionTwitter.new( - { 'urls' => ["invalid_url"], - 'interval' => 1, - 'retry' => 2 - } - ) - } - - its(:run) { should be_empty } - end -end diff --git a/spec/plugins/subscription/weather_spec.rb b/spec/plugins/subscription/weather_spec.rb deleted file mode 100644 index d362acb..0000000 --- a/spec/plugins/subscription/weather_spec.rb +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# Name:: Automatic::Plugin::Subscription::Weather -# Author: soramugi (More info: http://soramugi.net) -# Source Code:: https://github.com/id774/automaticruby -# License:: The GPL version 3, or LGPL version 3 (Dual License). -# Contact:: idnanashi@gmail.com -# Created:: May 12, 2013 -# Updated:: Aug 14, 2026 -# Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. - -require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') - -# livedoor Weather Hacks was terminated in 2020; -# doc/PLUGINS.md section 6.1 classifies SubscriptionWeather as Unsupported. -# -# The gem is not a dependency of this project, so this spec is skipped rather -# than stubbed; see doc/POLICY.md section 4. -return unless AutomaticSpec.plugin_available?('subscription/weather') - -describe Automatic::Plugin::SubscriptionWeather do - context "with empty zipcode" do - subject { - Automatic::Plugin::SubscriptionWeather.new( - {} - ) - } - - its(:run) { should be_empty } - end - - context "with zipcode whose return feed" do - subject { - Automatic::Plugin::SubscriptionWeather.new( - { 'zipcode' => '166-0003' } - ) - } - - its(:run) { should have(1).feed } - end - - context "with zipcode and day whose return feed" do - subject { - Automatic::Plugin::SubscriptionWeather.new( - { 'zipcode' => '166-0003', 'day' => 'tomorrow' } - ) - } - - its(:run) { should have(1).feed } - end - -end diff --git a/spec/plugins/subscription/xml_spec.rb b/spec/plugins/subscription/xml_spec.rb index e08b478..13f7b1f 100644 --- a/spec/plugins/subscription/xml_spec.rb +++ b/spec/plugins/subscription/xml_spec.rb @@ -75,7 +75,7 @@ subject { Automatic::Plugin::SubscriptionXml.new( { 'urls' => ["invalid_url"], - 'interval' => 1, + 'interval' => 0, 'retry' => 1 } ) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f9b73cd..1ef8671 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -96,7 +96,7 @@ module AutomaticSpec # need a service or a command as well as a gem, they are outside the # :plugins group for that reason, and their specs ask #plugin_available? # instead. - OPTIONAL_PLUGIN_GEMS = %w[activerecord feedbag nkf nokogiri sanitize sqlite3].freeze + OPTIONAL_PLUGIN_GEMS = %w[activerecord feedbag nokogiri sanitize sqlite3].freeze class << self # Load a plugin, or report that its dependency is absent. diff --git a/test/integration/test_add_pocket.yml b/test/integration/test_add_pocket.yml deleted file mode 100644 index 6f1c48d..0000000 --- a/test/integration/test_add_pocket.yml +++ /dev/null @@ -1,26 +0,0 @@ -global: - timezone: Asia/Tokyo - cache: - base: /tmp - log: - level: info - -plugins: - - module: SubscriptionFeed - config: - feeds: - - http://blog.id774.net/post/feed/ - - - module: StorePermalink - config: - db: test_instapaper.db - - - module: PublishPocket - config: - consumer_key: consumer_key - access_token: access_token - interval: 2 - retry: 2 - - #- module: PublishConsole - diff --git a/test/integration/test_chan_toru.yml b/test/integration/test_chan_toru.yml deleted file mode 100644 index 5a9f5c2..0000000 --- a/test/integration/test_chan_toru.yml +++ /dev/null @@ -1,21 +0,0 @@ -global: - timezone: Asia/Tokyo - cache: - base: /tmp - log: - level: none - -plugins: - - module: SubscriptionChanToru - config: - keyword: 'アニメ' - station: '地上波' - interval: 2 - retry: 2 - - - module: StorePermalink - config: - db: test_chan_toru.db - - #- module: PublishConsoleLink - diff --git a/test/integration/test_google_news.yml b/test/integration/test_google_news.yml deleted file mode 100644 index bcd8b43..0000000 --- a/test/integration/test_google_news.yml +++ /dev/null @@ -1,21 +0,0 @@ -global: - timezone: Asia/Tokyo - cache: - base: /tmp - log: - level: info - -plugins: - - module: SubscriptionFeed - config: - feeds: - - http://news.google.com/news?hl=ja&ned=us&ie=UTF-8&oe=UTF-8&output=rss - - - module: FilterGoogleNews - - - module: StorePermalink - config: - db: test_google_news.db - - - module: PublishConsole - diff --git a/test/integration/test_pocket.yml b/test/integration/test_pocket.yml deleted file mode 100644 index 1b0ce77..0000000 --- a/test/integration/test_pocket.yml +++ /dev/null @@ -1,22 +0,0 @@ -global: - timezone: Asia/Tokyo - cache: - base: /tmp - log: - level: warn - -plugins: - - module: SubscriptionPocket - config: - consumer_key: 'consumer_key' - access_token: 'access_token' - optional: - favorite: 3 - count: 2 - - - module: StorePermalink - config: - db: test_pocket.db - - #- module: PublishConsole - diff --git a/test/integration/test_twitter_search.yml b/test/integration/test_twitter_search.yml deleted file mode 100644 index 679e481..0000000 --- a/test/integration/test_twitter_search.yml +++ /dev/null @@ -1,22 +0,0 @@ -global: - timezone: Asia/Tokyo - cache: - base: /tmp - log: - level: none - -plugins: - - module: SubscriptionTwitterSearch - config: - consumer_key: 'consumer_key' - consumer_secret: 'consumer_secret' - oauth_token: 'oauth_token' - oauth_token_secret: 'oauth_token_secret' - search: 'ruby' - - - module: StorePermalink - config: - db: test_twitter_search.db - - #- module: PublishConsole - diff --git a/test/integration/test_weather.yml b/test/integration/test_weather.yml deleted file mode 100644 index 204e874..0000000 --- a/test/integration/test_weather.yml +++ /dev/null @@ -1,19 +0,0 @@ -global: - timezone: Asia/Tokyo - cache: - base: /tmp - log: - level: info - -plugins: - - module: SubscriptionWeather - config: - zipcode: 166-0003 - day: 'tomorrow' - - - module: StorePermalink - config: - db: test_weather.db - - #- module: PublishConsole -