<?php
// It may take a whils to crawl a site ...
require_once(dirname(__FILE__) . "/../../../adbcon.php");
set_time_limit(0);

require_once(dirname(__FILE__) . "/classes/phpcrawler.class.php");

class SelfSitemapGenerator extends PHPCrawler
{
    protected $sitemap_output_file;
    protected $robots_output_file = "";
    protected $robots_sitemap_url = "";

    public $counter = 0;

    public function setSitemapOutputFile($file, $robots = [])
    {
        $this->sitemap_output_file = $file;
        $this->robots_output_file = "";
        $this->robots_sitemap_url = "";

        if(
            isset($robots['location'])
            && isset($robots['ssl'])
            && isset($robots['domain'])
        )
        {
            $this->robots_sitemap_url = ($robots['ssl'] == 1 ? "https://" : "http://") . $robots['domain'] . "/sitemap.xml";
            $this->robots_output_file = $robots['location'];
        }

        if (file_exists($this->sitemap_output_file)) unlink($this->sitemap_output_file);

        file_put_contents($this->sitemap_output_file,
            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" .
            "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">\r\n",
            FILE_APPEND);
    }

    public function handleDocumentInfo($DocInfo)
    {
        // Just detect linebreak for output ("\n" in CLI-mode, otherwise "<br>").
        if (PHP_SAPI == "cli") $lb = "\n";
        else $lb = "<br />";

        $good = true;
        $neg = array(
            '+', '"', '\'', '$', '(', ')', 'this.href', '#', '%', 'filter='
        );

        $ur = $DocInfo->url;
        $httpCode = $DocInfo->http_status_code;


        foreach ($neg as $i => $n) {
            if (strpos($ur, $n) !== FALSE) $good = false;
        }


        if($httpCode == 200) {
            if ($good) {
                $this->counter += 1;

                echo $this->counter . ") " . $DocInfo->url . "" . $lb;
                /*echo '<span style="color:#ff0000;">' . "$httpCode : $ur " . $lb;
                echo 'Ref.URL <a href="' . $DocInfo->refering_link_raw . '" target="_blank">' . $DocInfo->referer_url . '</a>' . $lb;
                echo '<pre>Ref.Link.Code: ' . $DocInfo->refering_linkcode . '</pre>' . $lb;
                echo '</span>' . $lb;*/

                file_put_contents($this->sitemap_output_file, " <url>\r\n" .
                    "  <loc>" . $DocInfo->url . "</loc>\r\n" .
                    " </url>\r\n", FILE_APPEND);
            }
            else {
                echo "BAD: " . $ur . $lb;
                echo '<span style="color:#ff0000;">' . "$httpCode : $ur " . $lb;
                echo 'Ref.URL <a href="' . $DocInfo->refering_link_raw . '" target="_blank">' . $DocInfo->referer_url . '</a>' . $lb;
                echo '<pre>Ref.Link.Code: ' . $DocInfo->refering_linkcode . '</pre>' . $lb;
                echo '</span>' . $lb;
            }
        }
        else {
            echo '<span style="color:#ff0000;">' . "$httpCode : $ur " . $lb;
            echo 'Ref.URL <a href="' . $DocInfo->refering_link_raw . '" target="_blank">' . $DocInfo->referer_url . '</a>' . $lb;
            echo '<pre>Ref.Link.Code: ' . $DocInfo->refering_linkcode . '</pre>' . $lb;
            echo '</span>' . $lb;
        }
        //echo '<pre>', var_export($DocInfo, true), '</pre>';

        flush();
    }

    public function closeFile()
    {
        file_put_contents($this->sitemap_output_file, '</urlset>', FILE_APPEND);


        if($this->robots_output_file != "") {
            $robots_orig = __DIR__ . "/../../../aoriginal_files/robots.txt";
            $robots_contents = file_get_contents($robots_orig);
            $robots_contents .= "\n" . "Sitemap: " . $this->robots_sitemap_url;

            file_put_contents($this->robots_output_file, $robots_contents);
        }


    }
}


$argv = isset($_SERVER['argv']) ? $_SERVER['argv'] : null;
if (is_array($argv) && count($argv) > 0) {
    foreach ($argv as $arg) {
        parse_str($arg);
    }
}

if (isset($_REQUEST['sitename']) && trim($_REQUEST['sitename']) != "") {
    $sitename = $_REQUEST['sitename'];
}


if (isset($sitename) && trim($sitename) != "") {
    $domain_www = 'www.' . str_replace('www.', '', $sitename);
    //$q = "select * from domaincategories where domainname='$domain_www' and parent_id='0';";
    $q = "select d.* from domaincategories d where d.domainname='$domain_www' AND d.parent_id='0' and d.active='yes';";
    $r = mysql_query($q) or die("<strong>$q</strong>" . mysql_error());
    if ($r && mysql_num_rows($r) == 1) {
        $row = mysql_fetch_assoc($r);

        $ssl_enabled = false;
        $ssl_domain = intval($row['ssl_enabled']);

        if($ssl_domain != -1) {
            $ssl_enabled = true;
        }
        $addonname = $row['addonname'];
        $domain_folder = dirname(__FILE__) . "/../../../";
        $domain_sitemap_file = $domain_folder . 'sitemap.xml';

        $st = microtime(true);

        $crawler = new SelfSitemapGenerator();
        /*$crawler->excludeLinkSearchDocumentSections(
            PHPCrawlerLinkSearchDocumentSections::SCRIPT_SECTIONS |
            PHPCrawlerLinkSearchDocumentSections::JS_TRIGGERING_SECTIONS
        );*/
        $crawler->setSitemapOutputFile($domain_sitemap_file); // Set output-file
        $crawler->setURL( ($ssl_enabled ? 'https://' : 'http://') . $domain_www);

        $crawler->setStreamTimeout(5); // defaults to 2 seconds
        $crawler->setConnectionTimeout(10); // defaults to 5 seconds

        // Disable aggressive linksearch
        $crawler->enableAggressiveLinkSearch(false);

        $crawler->excludeLinkSearchDocumentSections(
            PHPCrawlerLinkSearchDocumentSections::ALL_SPECIAL_SECTIONS
        );


        $crawler->addContentTypeReceiveRule("#text/html#");
        $crawler->addURLFilterRule("#\.(jpg|jpeg|gif|png|php|swf|js|css|txt|svg|ico)$# i");
        $crawler->addURLFilterRule("#(click_count|account|gzout|module|javascript\#)# i");
        $crawler->addURLFilterRule("#(http%3A%2F%2F)# i");
        $crawler->addURLFilterRule("#(https%3A%2F%2F)# i");
        //$crawler->addURLFilterRule("#(\+|\'|\"|\(|\$)# i");

        // ... apply all other options and rules to the crawler

        //$crawler->setPageLimit(1000); // Just for testing
        //$crawler->goMultiProcessed(5); // Or use go() if you don't want multiple processes


        if (PHP_SAPI == "cli") {
            if (PHP_SAPI == "cli") echo "\n" . "Entered multi processes";
            //$crawler->goMultiProcessed(); // Or use go() if you don't want multiple processes
            //$crawler->goMultiProcessed(5, PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE); // Or use go() if you don't want multiple processes
            $crawler->go();
            //$crawler->go();
        } else {
            $crawler->go(); // Or use go() if you don't want multiple processes
            //$crawler->goMultiProcessed(200); // Or use go() if you don't want multiple processes
        }


        //$crawler->go(); // Or use go() if you don't want multiple processes
        $crawler->closeFile();
        // At the end, after the process is finished, we print a short
        // report (see method getProcessReport() for more information)
        $report = $crawler->getProcessReport();

        if (PHP_SAPI == "cli") $lb = "\n";
        else $lb = "<br />";

        $end = microtime(true);

        echo "Summary:" . $lb;

        echo "Time taken: " . ($end - $st) . " seconds" . $lb;

        echo "Links followed: " . $report->links_followed . $lb;
        echo "Documents received: " . $report->files_received . $lb;
        echo "Bytes received: " . $report->bytes_received . " bytes" . $lb;
        echo "Process runtime: " . $report->process_runtime . " sec" . $lb;

        $addon_folder = dirname(__FILE__) . "/../../..";
        $addon_folder_path = dirname(__FILE__) . "/../../../sitemap.xml";
        //copy($domain_sitemap_file, $addon_folder_path);

        $robots_sitemap_url = ($ssl_enabled ? "https://" : "http://") . $domain_www . "/sitemap.xml";
        $robots_orig = __DIR__ . "/../../../original_robots.txt";
        $robots_contents = file_get_contents($robots_orig);
        $robots_contents .= "\n" . "Sitemap: " . $robots_sitemap_url;

        echo "<br>Path: " . $addon_folder . "/robots.txt";
        echo '<pre>', $robots_contents, '</pre>';

        if( ($written = file_put_contents($addon_folder . "/robots.txt", $robots_contents)) !== false) {
            echo "<br>Written $written bytes";
        }
        else {
            echo "<br>The robots file has not been written.";
        }

    }




    exit();

} else {
    echo "No website to crawl.";
    exit();
}
?>