Example 11-6. fresh-links.php

require 'HTTP/Request.php';

function pc_link_extractor($s) {
    $a = array();
    if (preg_match_all('/<A\s+.*?HREF=[\"\']?([^\"\' >]*)[\"\']?[^>]*>(.*?)<\/A>/i',
                       $s,$matches,PREG_SET_ORDER)) {
        foreach($matches as $match) {
            array_push($a,array($match[1],$match[2]));
        }
    }
    return $a;
}

$url = $_SERVER['argv'][1];

// retrieve URLs in a loop to follow redirects 
$done = 0;
while (! $done) {
    $req = new HTTP_Request($url);
    $req->sendRequest();
    if ($response_code = $req->getResponseCode()) {
        if ((intval($response_code/100) == 3) &&
            ($location = $req->getResponseHeader('Location'))) {
            $url = $location;
        } else {
            $done = 1;
        }
    } else {
        return false;
    }
}

// compute base url from url
// this doesn't pay attention to a <base> tag in the page 
$base_url = preg_replace('{^(.*/)([^/]*)$}','\\1',$req->_url->getURL());

// keep track of the links we visit so we don't visit each more than once
$seen_links = array();

if ($body = $req->getResponseBody()) {
    $links = pc_link_extractor($body);
    foreach ($links as $link) {
        // skip https URLs
        if (preg_match('{^https://}',$link[0])) {
            continue;
        }
        // resolve relative links
        if (! (preg_match('{^(http|mailto):}',$link[0]))) {
            $link[0] = $base_url.$link[0];
        }
        // skip this link if we've seen it already
        if ($seen_links[$link[0]]) {
            continue;
        } 
        
        // mark this link as seen
        $seen_links[$link[0]] = true;

        // print the link we're visiting
        print $link[0].': ';
        flush();
        
        // visit the link
        $req2 = new HTTP_Request($link[0],
                                 array('method' => HTTP_REQUEST_METHOD_HEAD));
        $now = time();
        $req2->sendRequest();
        $response_code = $req2->getResponseCode();
        
        // if the retrieval is successful
        if ($response_code == 200) {
            // get the Last-Modified header
            if ($lm = $req2->getResponseHeader('Last-Modified')) {
                $lm_utc = strtotime($lm);
            } else {
                // or set Last-Modified to now
                $lm_utc = $now;
            }
            print strftime('%c',$lm_utc);
        } else {
            // otherwise, print the response code
            print $response_code;
        }
        print "\n";
    }
}