structured_scraper/scrape.php

32 lines
757 B
PHP
Raw Normal View History

2022-01-06 17:20:40 +01:00
<?php
require __DIR__.'/vendor/autoload.php';
use Scraping\Scraper;
2022-01-08 20:30:33 +01:00
use Goutte\Client;
2022-01-06 17:20:40 +01:00
2022-01-08 20:30:33 +01:00
// Usage
if (count($argv) == 1) {
die('Usage: '.$argv[0]." <config.php>\n");
} else {
require __DIR__.'/'.$argv[1];
}
$client = new Client();
2022-01-06 17:20:40 +01:00
$scraper = new Scraper();
2022-01-08 20:30:33 +01:00
// Configure object
$scraper->allowedMimetypes = $allowedMimetypes;
$scraper->link_rules = $link_rules;
$scraper->scrape($start_urls, 0);
// Download documents
foreach ($scraper->results as $url => $scraped_obj) {
if ($scraped_obj['is-downloadable']) {
$client->request('GET', $url);
print '[*] Downloading '.$scraped_obj['filename']."\n";
file_put_contents( $download_dir.'/'.$scraped_obj['filename'], $client->getResponse()->getContent() );
}
}