444: def fetch_page(uri, request, cur_page=current_page(), request_data=[])
445: raise "unsupported scheme" unless ['http', 'https'].include?(uri.scheme)
446:
447: log.info("#{ request.class }: #{ request.path }") if log
448:
449: page = nil
450:
451: cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
452: :connection => nil,
453: :keep_alive_options => {},
454: })
455: http_obj = cache_obj[:connection]
456: if http_obj.nil? || ! http_obj.started?
457: http_obj = cache_obj[:connection] =
458: Net::HTTP.new( uri.host,
459: uri.port,
460: @proxy_addr,
461: @proxy_port,
462: @proxy_user,
463: @proxy_pass
464: )
465: cache_obj[:keep_alive_options] = {}
466:
467:
468: http_obj.open_timeout = @open_timeout if @open_timeout
469: http_obj.read_timeout = @read_timeout if @read_timeout
470: end
471:
472: if uri.scheme == 'https' && ! http_obj.started?
473: http_obj.use_ssl = true
474: http_obj.verify_mode = OpenSSL::SSL::VERIFY_NONE
475: if @ca_file
476: http_obj.ca_file = @ca_file
477: http_obj.verify_mode = OpenSSL::SSL::VERIFY_PEER
478: end
479: if @cert && @key
480: http_obj.cert = OpenSSL::X509::Certificate.new(::File.read(@cert))
481: http_obj.key = OpenSSL::PKey::RSA.new(::File.read(@key), @pass)
482: end
483: end
484:
485:
486:
487:
488: if @keep_alive && http_obj.started?
489: opts = cache_obj[:keep_alive_options]
490: if((opts[:timeout] &&
491: Time.now.to_i - cache_obj[:last_request_time] > opts[:timeout].to_i) ||
492: opts[:max] && opts[:max].to_i == 1)
493:
494: log.debug('Finishing stale connection') if log
495: http_obj.finish
496:
497: end
498: end
499:
500: http_obj.start unless http_obj.started?
501:
502: request = set_headers(uri, request, cur_page)
503:
504:
505: if log
506: request.each_header do |k, v|
507: log.debug("request-header: #{ k } => #{ v }")
508: end
509: end
510:
511: cache_obj[:last_request_time] = Time.now.to_i
512:
513:
514: response = http_obj.request(request, *request_data) {|response|
515:
516: body = StringIO.new
517: total = 0
518: response.read_body { |part|
519: total += part.length
520: body.write(part)
521: log.debug("Read #{total} bytes") if log
522: }
523: body.rewind
524:
525: response.each_header { |k,v|
526: log.debug("response-header: #{ k } => #{ v }")
527: } if log
528:
529: content_type = nil
530: unless response['Content-Type'].nil?
531: data = response['Content-Type'].match(/^([^;]*)/)
532: content_type = data[1].downcase unless data.nil?
533: end
534:
535: response_body =
536: if encoding = response['Content-Encoding']
537: case encoding.downcase
538: when 'gzip'
539: log.debug('gunzip body') if log
540: Zlib::GzipReader.new(body).read
541: when 'x-gzip'
542: body.read
543: else
544: raise 'Unsupported content encoding'
545: end
546: else
547: body.read
548: end
549:
550:
551: page = @pluggable_parser.parser(content_type).new(
552: uri,
553: response,
554: response_body,
555: response.code
556: ) { |parser|
557: parser.mech = self if parser.respond_to? :mech=
558: if parser.respond_to?(:watch_for_set=) && @watch_for_set
559: parser.watch_for_set = @watch_for_set
560: end
561: }
562:
563: }
564:
565:
566: if keep_alive_info = response['keep-alive']
567: keep_alive_info.split(/,\s*/).each do |option|
568: k, v = option.split(/=/)
569: cache_obj[:keep_alive_options] ||= {}
570: cache_obj[:keep_alive_options][k.intern] = v
571: end
572: end
573:
574: (response.get_fields('Set-Cookie')||[]).each do |cookie|
575: Cookie::parse(uri, cookie, log) { |c|
576: log.debug("saved cookie: #{c}") if log
577: @cookie_jar.add(uri, c)
578: }
579: end
580:
581: log.info("status: #{ page.code }") if log
582:
583: res_klass = Net::HTTPResponse::CODE_TO_OBJ[page.code.to_s]
584:
585: if follow_meta_refresh && page.respond_to?(:meta) &&
586: (redirect = page.meta.first)
587: return redirect.click
588: end
589:
590: return page if res_klass <= Net::HTTPSuccess
591:
592: if res_klass == Net::HTTPNotModified
593: log.debug("Got cached page") if log
594: return visited_page(uri)
595: elsif res_klass <= Net::HTTPRedirection
596: return page unless follow_redirect?
597: log.info("follow redirect to: #{ response['Location'] }") if log
598: from_uri = page.uri
599: abs_uri = to_absolute_uri(response['Location'].to_s, page)
600: page = fetch_page(abs_uri, fetch_request(abs_uri), page)
601: @history.push(page, from_uri)
602: return page
603: elsif res_klass <= Net::HTTPUnauthorized
604: raise ResponseCodeError.new(page) unless @user || @password
605: raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host)
606: if response['www-authenticate'] =~ /Digest/i
607: @auth_hash[uri.host] = :digest
608: @digest = response['www-authenticate']
609: else
610: @auth_hash[uri.host] = :basic
611: end
612: return fetch_page( uri,
613: fetch_request(uri, request.method.downcase.to_sym),
614: cur_page,
615: request_data
616: )
617: end
618:
619: raise ResponseCodeError.new(page), "Unhandled response", caller
620: end
