Completamento scraping con dowload

config.example.php

@@ -1,10 +1,24 @@
 <?php
 
-$start_url = 'https://www.iltuocomune.it/amministrazione-trasparente-url';
+// URL da cui iniziare lo scraping
+$start_urls = [
+    'https://www.iltuocomune.it/amministrazione-trasparente-url'
+];
 
-$filters = [
+// Regole di estrazione link.
+// Ogni regola è applicata alle pagine trovate in base al livello di profondità.
+// Esempio: la prima regola viene applicata alle start_urls, la seconda alle
+// sotto-pagine, e così via.
+$link_rules = [
     '//td[@class="actions"]/a[@title="Visualizza "]',
     '//table[@class="allegati"]/a[@title="Download versione non firmata"]'
 ];
 
+// Mimetype da salvare, con associazione mime-type => estensione con cui salvare
+// i file
+$allowedMimetypes = [
+    'application/pdf' => 'pdf'
+];
+
+// Directory dove salvare i file trovati
 $download_dir = __DIR__.'/pdf';

example/pagina1/sub1.1/index.html

@@ -1,2 +1,3 @@
 <a href="subsub1.1.1">Subsublink 1</a><br>
 <a href="subsub1.1.2">Subsublink 2</a><br>
+<a href="test.pdf">Document</a><br>

scrape.php

@@ -1,10 +1,32 @@
 <?php
 
 require __DIR__.'/vendor/autoload.php';
-require __DIR__.'/config.php';
 
 use Scraping\Scraper;
+use Goutte\Client;
 
+// Usage
+if (count($argv) == 1) {
+    die('Usage: '.$argv[0]." <config.php>\n");
+} else {
+    require __DIR__.'/'.$argv[1];
+}
+
+$client = new Client();
 $scraper = new Scraper();
 
-$scraper->scrape('GET', $start_urls, $filters, 0, $download_dir);
+// Configure object
+$scraper->allowedMimetypes = $allowedMimetypes;
+$scraper->link_rules = $link_rules;
+
+$scraper->scrape($start_urls, 0);
+
+// Download documents
+foreach ($scraper->results as $url => $scraped_obj) {
+    if ($scraped_obj['is-downloadable']) {
+        $client->request('GET', $url);
+        
+        print '[*] Downloading '.$scraped_obj['filename']."\n";
+        file_put_contents( $download_dir.'/'.$scraped_obj['filename'], $client->getResponse()->getContent() );
+    }
+}

src/Scraping/Scraper.php

@@ -6,45 +6,103 @@ use \Goutte\Client;
 
 class Scraper
 {
-    public $links = [];
+    public $results = [];
+    public $allowedMimetypes;
+    public $link_rules;
 
-    public static function scrape($method, $urls, $filters, $level, $download_dir){
+    /**
+     * Scrapes specified URLs
+     * 
+     * @param array     $urls
+     * @param int       $level
+     */
+    public function scrape(array $urls, int $level){
         $client = new Client();
 
         $sub_urls = [];
 
         foreach ($urls as $url) {
-            $crawler = $client->request($method, $url);
+            $crawler = $client->request('GET', $url);
 
-            //if (strstr($client->getResponse()->getHeaders()['content-type'][0], 'text/html')) {
-                print '['.$level.'] '.$url."\n";
+            if( $url == 'http://localhost/example/pagina1/sub1.1/test.pdf' ){
+                $a = 1;
+            }
 
-                $new_urls = $crawler->filterXPath($filters[$level])->each(function ($node) use ($sub_urls) {
+            $scraped_obj[$url]['content-type'] = $this->getContentType($client);
+            $scraped_obj[$url]['is-downloadable'] = $this->isDownloadable($client);
+            $scraped_obj[$url]['filename'] = ( $scraped_obj[$url]['is-downloadable'] ? $this->getFilename($client) : '' );
+
+            $this->results = array_merge( $this->results, $scraped_obj );
+
+            $new_urls = $crawler->filterXPath($this->link_rules[$level])->each(function ($node) {
                 return $node->link()->getUri();
             });
 
             $sub_urls = array_merge( $sub_urls, $new_urls );
-            /*
-            } else {
+        }
+
+        if ($level++ < count($this->link_rules) && !empty($sub_urls)) {
+            $this->scrape($sub_urls, $level);
+        }
+    }
+
+    /**
+     * Returns if URL is downloadable
+     * 
+     * @return boolean
+     * 
+     * @param Client    $client
+     */
+    public function isDownloadable(Client $client){
+        foreach ($this->allowedMimetypes as $mimetype) {
+            if (isset($client->getResponse()->getHeaders()['content-type'][0])) {
+                if (strstr($client->getResponse()->getHeaders()['content-type'][0], $mimetype)) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Get filename of http response based on URL or content-disposition header
+     * 
+     * @return string
+     * 
+     * @param Client    $client
+     */
+    public function getFilename(Client $client){
+        $filename = basename( $client->getRequest()->getUri() );
+
+        // Try to get filename from content-disposition
+        if (isset($client->getResponse()->getHeaders()['content-disposition'][0])) {
             $content_disposition = $client->getResponse()->getHeaders()['content-disposition'][0];
             
-                $filename = time();
-                
             if (preg_match('/filename="([^"]+)"/', $content_disposition, $m)) {
-                    $filename = $m[1];
+                $filename = $m[1].'-'.$this->allowedMimetypes[ $this->getContentType($client) ];
+            } else {
+                $filename = time();
+            }
         }
 
-                file_put_contents( $download_dir.'/'.$filename, $client->getResponse()->getContent() );
+        return $filename;
     }
+
+    /**
+     * Get content type of http response
+     * 
+     * @return string
+     * 
+     * @param Client    $client
      */
+    public static function getContentType(Client $client){
+        $content_type = '';
+
+        if (isset($client->getResponse()->getHeaders()['content-type'][0])) {
+            $content_type = $client->getResponse()->getHeaders()['content-type'][0];
         }
 
-        if($level==3){
-            $a=1;
-        }
-
-        if ($level++ < count($filters) && !empty($sub_urls)) {
-            self::scrape($method, $sub_urls, $filters, $level, $download_dir);
-        }
+        return $content_type;
     }
 }