33 lines
783 B
PHP
33 lines
783 B
PHP
<?php
|
|
|
|
require __DIR__.'/vendor/autoload.php';
|
|
|
|
use Scraping\Scraper;
|
|
use Goutte\Client;
|
|
|
|
// Usage
|
|
if (count($argv) == 1) {
|
|
die('Usage: '.$argv[0]." <config.php>\n");
|
|
} else {
|
|
require __DIR__.'/'.$argv[1];
|
|
}
|
|
|
|
$client = new Client();
|
|
$scraper = new Scraper();
|
|
|
|
// Configure object
|
|
$scraper->allowedMimetypes = $allowedMimetypes;
|
|
$scraper->link_rules = $link_rules;
|
|
$scraper->debug = $debug;
|
|
|
|
$scraper->scrape($start_urls, 0);
|
|
|
|
// Download documents
|
|
foreach ($scraper->results as $url => $scraped_obj) {
|
|
if ($scraped_obj['is-downloadable']) {
|
|
$client->request('GET', $url);
|
|
|
|
print '[*] Downloading '.$scraped_obj['filename']."\n";
|
|
file_put_contents( $download_dir.'/'.$scraped_obj['filename'], $client->getResponse()->getContent() );
|
|
}
|
|
} |