structured_scraper/scrape.php

<?php

require __DIR__.'/vendor/autoload.php';

use Scraping\Scraper;
use Goutte\Client;

// Usage
if (count($argv) == 1) {
    die('Usage: '.$argv[0]." <config.php>\n");
} else {
    require __DIR__.'/'.$argv[1];
}

$client = new Client();
$scraper = new Scraper();

// Configure object
$scraper->allowedMimetypes = $allowedMimetypes;
$scraper->link_rules = $link_rules;
$scraper->debug = $debug;

$scraper->scrape($start_urls, 0);

// Download documents
foreach ($scraper->results as $url => $scraped_obj) {
    if ($scraped_obj['is-downloadable']) {
        $client->request('GET', $url);
        
        print '[*] Downloading '.$scraped_obj['filename']."\n";
        file_put_contents( $download_dir.'/'.$scraped_obj['filename'], $client->getResponse()->getContent() );
    }
}