Files
growstuff/spec/services/remote_api_client_spec.rb
T
Brenda WallaceandClaude Sonnet 5 23e83b1a5f Add rake tasks to pull crops and a member's data from production (#4803)
* Add rake tasks to pull crops and a member's data from production

crops:pull_from_production copies approved crops, and
members:pull_from_production copies one member's gardens, plantings and
planting photos, from growstuff.org into a local database. Both refuse to
run in a production environment.

They are built to be gentle on a server that has been struggling: one
request at a time with a pause between requests, exponential backoff
(honouring Retry-After) on timeouts, 429s and 5xx, and a hard stop rather
than endless retries. The client also stops if a server keeps sending the
same page, and will not follow a next link to another host.

Gardens and plantings come from the site's own JSON, which carries every
column; the JSON:API only exposes a garden's name, and its related
resource endpoints return the wrong records. Only the crops that a
member's plantings use are fetched. Flickr photos are stored as records
pointing at Flickr's image URLs, as on production, so nothing is
downloaded from Flickr.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Add ACTIVE_ONLY option to the member import

ACTIVE_ONLY=true leaves out the site's all=1 flag, so production lists only
the member's active gardens and current (not finished, not failed)
plantings. The server does the filtering, so it also means fewer requests,
and no crops or photos are fetched for plantings that are skipped.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-20 14:06:01 +09:30

123 lines
4.2 KiB
Ruby

# frozen_string_literal: true
require 'rails_helper'
describe RemoteApiClient, type: :service do
let(:stubs) { Faraday::Adapter::Test::Stubs.new }
let(:connection) { Faraday.new(url: 'https://source.test') { |f| f.adapter :test, stubs } }
let(:sleeps) { [] }
let(:client) do
described_class.new(connection: connection, delay: 1, backoff: 5, out: StringIO.new,
sleeper: ->(seconds) { sleeps << seconds })
end
around { |example| VCR.turned_off(&example) }
def json(body, status = 200)
[status, {}, body.to_json]
end
describe '#each_page (JSON:API)' do
it 'follows links.next verbatim, whatever paging scheme the server uses' do
stubs.get('/things') do |env|
offset = env.params.dig('page', 'offset').to_i
next json(data: [{ id: '1' }], links: { next: 'https://source.test/things?page[offset]=1' }) if offset.zero?
json(data: [{ id: '2' }], links: {})
end
pages = []
client.each_page('/things') { |resources, _page| pages << resources.pluck('id') }
expect(pages).to eq [['1'], ['2']]
expect(sleeps).to eq [1]
end
it 'stops rather than loop when the server keeps sending the same next link' do
stubs.get('/things') { json(data: [{ id: '1' }], links: { next: 'https://source.test/things?page[offset]=1' }) }
expect { client.each_page('/things') { |_resources, _page| nil } }
.to raise_error(described_class::Aborted, /same next page/)
end
it 'refuses to follow a next link to another host' do
stubs.get('/things') { json(data: [{ id: '1' }], links: { next: 'https://elsewhere.test/things' }) }
expect { client.each_page('/things') { |_resources, _page| nil } }
.to raise_error(described_class::Aborted, /another host/)
end
it 'aborts on a response without data' do
stubs.get('/things') { json(errors: [{ status: '500' }]) }
expect { client.each_page('/things') { |_resources, _page| nil } }
.to raise_error(described_class::Aborted, /unexpected response/)
end
end
describe 'request headers' do
it 'identifies itself and asks for JSON:API, which production insists on' do
client = described_class.new(source_url: 'https://source.test', out: StringIO.new, sleeper: ->(_seconds) {})
headers = client.send(:build_connection, 'https://source.test').headers
expect(headers['Accept']).to eq 'application/vnd.api+json'
expect(headers['User-Agent']).to include('Growstuff')
end
end
describe '#get_resource' do
it 'returns the resource' do
stubs.get('/things/1') { json(data: { id: '1', type: 'things' }) }
expect(client.get_resource('/things/1')).to include('id' => '1')
end
it 'returns nil for a missing resource instead of aborting' do
stubs.get('/things/9') { [404, {}, ''] }
expect(client.get_resource('/things/9')).to be_nil
end
end
describe '#each_site_page (plain JSON)' do
it 'pages with ?page=N until an empty page' do
stubs.get('/things.json') do |env|
rows = { '1' => [{ 'id' => 1 }, { 'id' => 2 }], '2' => [{ 'id' => 3 }] }
json(rows.fetch(env.params['page'], []))
end
pages = []
client.each_site_page('/things.json') { |rows, page| pages << [page, rows.pluck('id')] }
expect(pages).to eq [[1, [1, 2]], [2, [3]]]
end
it 'stops rather than loop when the server ignores ?page=' do
stubs.get('/things.json') { json([{ 'id' => 1 }]) }
expect { client.each_site_page('/things.json') { |_rows, _page| nil } }
.to raise_error(described_class::Aborted, /same page twice/)
end
it 'treats a 404 after the first page as the end of the list' do
stubs.get('/things.json') do |env|
env.params['page'] == '1' ? json([{ 'id' => 1 }]) : [404, {}, '']
end
pages = []
client.each_site_page('/things.json') { |rows, _page| pages << rows }
expect(pages.size).to eq 1
end
it 'can be limited to max_pages' do
stubs.get('/things.json') { |env| json([{ 'id' => env.params['page'].to_i }]) }
pages = []
client.each_site_page('/things.json', max_pages: 1) { |rows, _page| pages << rows }
expect(pages.size).to eq 1
end
end
end