419: def fetch_page(params)
420: options = {
421: :request => nil,
422: :response => nil,
423: :connection => nil,
424: :referer => current_page(),
425: :uri => nil,
426: :verb => :get,
427: :agent => self,
428: :redirects => 0,
429: :params => [],
430: :headers => {},
431: }.merge(params)
432:
433: before_connect = Chain.new([
434: Chain::URIResolver.new(@scheme_handlers),
435: Chain::ParameterResolver.new,
436: Chain::RequestResolver.new,
437: Chain::ConnectionResolver.new(
438: @connection_cache,
439: @keep_alive,
440: @proxy_addr,
441: @proxy_port,
442: @proxy_user,
443: @proxy_pass
444: ),
445: Chain::SSLResolver.new(@ca_file, @verify_callback, @cert, @key, @pass),
446: Chain::AuthHeaders.new(@auth_hash, @user, @password, @digest),
447: Chain::HeaderResolver.new(
448: @keep_alive,
449: @keep_alive_time,
450: @cookie_jar,
451: @user_agent,
452: {}
453: ),
454: Chain::CustomHeaders.new,
455: @pre_connect_hook,
456: ])
457: before_connect.handle(options)
458:
459: uri = options[:uri]
460: request = options[:request]
461: cur_page = options[:referer]
462: request_data = options[:params]
463: redirects = options[:redirects]
464: http_obj = options[:connection]
465:
466:
467: if( (page = visited_page(uri)) && page.response['Last-Modified'] )
468: request['If-Modified-Since'] = page.response['Last-Modified']
469: end if(@conditional_requests)
470:
471:
472: http_obj.open_timeout = @open_timeout if @open_timeout
473: http_obj.read_timeout = @read_timeout if @read_timeout
474: http_obj.start unless http_obj.started?
475:
476:
477: log.info("#{ request.class }: #{ request.path }") if log
478: request.each_header do |k, v|
479: log.debug("request-header: #{ k } => #{ v }")
480: end if log
481:
482:
483: attempts = 0
484: begin
485: response = http_obj.request(request, *request_data) { |r|
486: connection_chain = Chain.new([
487: Chain::ResponseReader.new(r),
488: Chain::BodyDecodingHandler.new,
489: ])
490: connection_chain.handle(options)
491: }
492: rescue EOFError, Errno::ECONNRESET, Errno::EPIPE => x
493: log.error("Rescuing EOF error") if log
494: http_obj.finish
495: raise x if attempts >= 2
496: request.body = nil
497: http_obj.start
498: attempts += 1
499: retry
500: end
501:
502: after_connect = Chain.new([
503: @post_connect_hook,
504: Chain::ResponseBodyParser.new(@pluggable_parser, @watch_for_set),
505: Chain::ResponseHeaderHandler.new(@cookie_jar, @connection_cache),
506: ])
507: after_connect.handle(options)
508:
509: res_klass = options[:res_klass]
510: response_body = options[:response_body]
511: page = options[:page]
512:
513: log.info("status: #{ page.code }") if log
514:
515: if follow_meta_refresh
516: redirect_uri = nil
517: referer = page
518: if (page.respond_to?(:meta) && (redirect = page.meta.first))
519: redirect_uri = redirect.uri.to_s
520: sleep redirect.node['delay'].to_f
521: referer = Page.new(nil, {'content-type'=>'text/html'})
522: elsif refresh = response['refresh']
523: delay, redirect_uri = Page::Meta.parse(refresh, uri)
524: raise StandardError, "Invalid refresh http header" unless delay
525: if redirects + 1 > redirection_limit
526: raise RedirectLimitReachedError.new(page, redirects)
527: end
528: sleep delay.to_f
529: end
530: if redirect_uri
531: @history.push(page, page.uri)
532: return fetch_page(
533: :uri => redirect_uri,
534: :referer => referer,
535: :params => [],
536: :verb => :get,
537: :redirects => redirects + 1
538: )
539: end
540: end
541:
542: return page if res_klass <= Net::HTTPSuccess
543:
544: if res_klass == Net::HTTPNotModified
545: log.debug("Got cached page") if log
546: return visited_page(uri) || page
547: elsif res_klass <= Net::HTTPRedirection
548: return page unless follow_redirect?
549: log.info("follow redirect to: #{ response['Location'] }") if log
550: from_uri = page.uri
551: raise RedirectLimitReachedError.new(page, redirects) if redirects + 1 > redirection_limit
552: redirect_verb = options[:verb] == :head ? :head : :get
553: page = fetch_page( :uri => response['Location'].to_s,
554: :referer => page,
555: :params => [],
556: :verb => redirect_verb,
557: :redirects => redirects + 1
558: )
559: @history.push(page, from_uri)
560: return page
561: elsif res_klass <= Net::HTTPUnauthorized
562: raise ResponseCodeError.new(page) unless @user || @password
563: raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host)
564: if response['www-authenticate'] =~ /Digest/i
565: @auth_hash[uri.host] = :digest
566: if response['server'] =~ /Microsoft-IIS/
567: @auth_hash[uri.host] = :iis_digest
568: end
569: @digest = response['www-authenticate']
570: else
571: @auth_hash[uri.host] = :basic
572: end
573: return fetch_page( :uri => uri,
574: :referer => cur_page,
575: :verb => request.method.downcase.to_sym,
576: :params => request_data,
577: :headers => options[:headers]
578: )
579: end
580:
581: raise ResponseCodeError.new(page), "Unhandled response", caller
582: end
