commit 7b5280733707292acc4ad0576130b8952f92d147
parent 3c0a3e09b8a32c2dcfe000f097c2f9b303797eb0
Author: markseu <mark2011@mayberg.se>
Date: Sun, 22 Apr 2018 11:42:08 +0200
Updated system, spring remix with broken link checker
Diffstat:
2 files changed, 243 insertions(+), 40 deletions(-)
diff --git a/system/plugins/command.php b/system/plugins/command.php
@@ -1,13 +1,14 @@
<?php
// Command plugin, https://github.com/datenstrom/yellow-plugins/tree/master/command
-// Copyright (c) 2013-2017 Datenstrom, https://datenstrom.se
+// Copyright (c) 2013-2018 Datenstrom, https://datenstrom.se
// This file may be used and distributed under the terms of the public license.
class YellowCommand
{
- const VERSION = "0.7.3";
+ const VERSION = "0.7.4";
var $yellow; //access to API
var $files; //number of files
+ var $links; //number of links
var $errors; //number of errors
var $locationsArgs; //locations with location arguments detected
var $locationsArgsPagination; //locations with pagination arguments detected
@@ -26,6 +27,7 @@ class YellowCommand
{
case "": $statusCode = $this->helpCommand(); break;
case "build": $statusCode = $this->buildCommand($args); break;
+ case "check": $statusCode = $this->checkCommand($args); break;
case "clean": $statusCode = $this->cleanCommand($args); break;
case "version": $statusCode = $this->versionCommand($args); break;
default: $statusCode = 0;
@@ -37,6 +39,7 @@ class YellowCommand
function onCommandHelp()
{
$help .= "build [DIRECTORY LOCATION]\n";
+ $help .= "check [DIRECTORY LOCATION]\n";
$help .= "clean [DIRECTORY LOCATION]\n";
$help .= "version\n";
return $help;
@@ -51,7 +54,7 @@ class YellowCommand
return 200;
}
- // Build static files
+ // Build static website
function buildCommand($args)
{
$statusCode = 0;
@@ -60,7 +63,7 @@ class YellowCommand
{
if($this->checkStaticConfig())
{
- $statusCode = $this->buildStatic($path, $location);
+ $statusCode = $this->buildStaticFiles($path, $location);
} else {
$statusCode = 500;
$this->files = 0; $this->errors = 1;
@@ -68,8 +71,7 @@ class YellowCommand
echo "ERROR building files: Please configure StaticUrl in file '$fileName'!\n";
}
echo "Yellow $command: $this->files file".($this->files!=1 ? 's' : '');
- echo ", $this->errors error".($this->errors!=1 ? 's' : '');
- echo ", status $statusCode\n";
+ echo ", $this->errors error".($this->errors!=1 ? 's' : '')."\n";
} else {
$statusCode = 400;
echo "Yellow $command: Invalid arguments\n";
@@ -77,36 +79,41 @@ class YellowCommand
return $statusCode;
}
- // Build static files and additional locations
- function buildStatic($path, $location)
+ // Build static files
+ function buildStaticFiles($path, $locationFilter)
{
$path = rtrim(empty($path) ? $this->yellow->config->get("staticDir") : $path, '/');
- $this->files = $this->errors = $statusCode = 0;
+ $this->files = $this->errors = 0;
$this->locationsArgs = $this->locationsArgsPagination = array();
- if(empty($location))
+ $statusCode = empty($locationFilter) ? $this->cleanStaticFiles($path, $locationFilter) : 200;
+ $staticUrl = $this->yellow->config->get("staticUrl");
+ list($scheme, $address, $base) = $this->yellow->lookup->getUrlInformation($staticUrl);
+ foreach($this->getContentLocations() as $location)
+ {
+ if(!preg_match("#^$base$locationFilter#", $location)) continue;
+ $statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
+ }
+ foreach($this->locationsArgs as $location)
{
- $statusCode = $this->cleanStatic($path, $location);
- foreach($this->getContentLocations() as $location)
+ if(!preg_match("#^$base$locationFilter#", $location)) continue;
+ $statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
+ }
+ foreach($this->locationsArgsPagination as $location)
+ {
+ if(!preg_match("#^$base$locationFilter#", $location)) continue;
+ if(substru($location, -1)!=$this->yellow->toolbox->getLocationArgsSeparator())
{
- $statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
+ $statusCode = max($statusCode, $this->buildStaticFile($path, $location, false, true));
}
- foreach($this->locationsArgs as $location)
+ for($pageNumber=2; $pageNumber<=999; ++$pageNumber)
{
- $statusCode = max($statusCode, $this->buildStaticFile($path, $location, true));
- }
- foreach($this->locationsArgsPagination as $location)
- {
- if(substru($location, -1)!=$this->yellow->toolbox->getLocationArgsSeparator())
- {
- $statusCode = max($statusCode, $this->buildStaticFile($path, $location, false, true));
- }
- for($pageNumber=2; $pageNumber<=999; ++$pageNumber)
- {
- $statusCodeLocation = $this->buildStaticFile($path, $location.$pageNumber, false, true);
- $statusCode = max($statusCode, $statusCodeLocation);
- if($statusCodeLocation==100) break;
- }
+ $statusCodeLocation = $this->buildStaticFile($path, $location.$pageNumber, false, true);
+ $statusCode = max($statusCode, $statusCodeLocation);
+ if($statusCodeLocation==100) break;
}
+ }
+ if(empty($locationFilter))
+ {
foreach($this->getMediaLocations() as $location)
{
$statusCode = max($statusCode, $this->buildStaticFile($path, $location));
@@ -116,8 +123,6 @@ class YellowCommand
$statusCode = max($statusCode, $this->buildStaticFile($path, $location));
}
$statusCode = max($statusCode, $this->buildStaticFile($path, "/error/", false, false, true));
- } else {
- $statusCode = $this->buildStaticFile($path, $location);
}
return $statusCode;
}
@@ -232,6 +237,161 @@ class YellowCommand
}
}
}
+
+ // Check static files for broken links
+ function checkCommand($args)
+ {
+ $statusCode = 0;
+ list($command, $path, $location) = $args;
+ if(empty($location) || $location[0]=='/')
+ {
+ if($this->checkStaticConfig())
+ {
+ $statusCode = $this->checkStaticFiles($path, $location);
+ } else {
+ $statusCode = 500;
+ $this->files = $this->links = 0;
+ $fileName = $this->yellow->config->get("configDir").$this->yellow->config->get("configFile");
+ echo "ERROR checking files: Please configure StaticUrl in file '$fileName'!\n";
+ }
+ echo "Yellow $command: $this->files file".($this->files!=1 ? 's' : '');
+ echo ", $this->links link".($this->links!=1 ? 's' : '')."\n";
+ } else {
+ $statusCode = 400;
+ echo "Yellow $command: Invalid arguments\n";
+ }
+ return $statusCode;
+ }
+
+ // Check static files
+ function checkStaticFiles($path, $locationFilter)
+ {
+ $path = rtrim(empty($path) ? $this->yellow->config->get("staticDir") : $path, '/');
+ $this->files = $this->links = 0;
+ $regex = "/^[^.]+$|".$this->yellow->config->get("staticDefaultFile")."$/";
+ $fileNames = $this->yellow->toolbox->getDirectoryEntriesRecursive($path, $regex, false, false);
+ list($statusCodeFiles, $links) = $this->analyseStaticFiles($path, $locationFilter, $fileNames);
+ list($statusCodeLinks, $broken, $redirect) = $this->analyseLinks($path, $links);
+ if($statusCodeLinks!=200)
+ {
+ $this->showLinks($broken, "Broken links");
+ $this->showLinks($redirect, "Redirect links");
+ }
+ return max($statusCodeFiles, $statusCodeLinks);
+ }
+
+ // Analyse static files, detect links
+ function analyseStaticFiles($path, $locationFilter, $fileNames)
+ {
+ $statusCode = 200;
+ $links = array();
+ if(!empty($fileNames))
+ {
+ $staticUrl = $this->yellow->config->get("staticUrl");
+ list($scheme, $address, $base) = $this->yellow->lookup->getUrlInformation($staticUrl);
+ foreach($fileNames as $fileName)
+ {
+ if(is_readable($fileName))
+ {
+ $locationSource = $this->getStaticLocation($path, $fileName);
+ if(!preg_match("#^$base$locationFilter#", $locationSource)) continue;
+ $fileData = $this->yellow->toolbox->readFile($fileName);
+ preg_match_all("/<(.*?)href=\"([^\"]+)\"(.*?)>/i", $fileData, $matches);
+ foreach($matches[2] as $match)
+ {
+ $location = rawurldecode($match);
+ if(preg_match("/^(.*?)#(.*)$/", $location, $tokens)) $location = $tokens[1];
+ if(preg_match("/^(\w+):\/\/([^\/]+)(.*)$/", $location, $matches))
+ {
+ $url = $location.(empty($matches[3]) ? "/" : "");
+ if(!is_null($links[$url])) $links[$url] .= ",";
+ $links[$url] .= $locationSource;
+ if(defined("DEBUG") && DEBUG>=2) echo "YellowCommand::analyseStaticFiles detected url:$url<br/>\n";
+ } else if($location[0]=='/') {
+ $url = "$scheme://$address$location";
+ if(!is_null($links[$url])) $links[$url] .= ",";
+ $links[$url] .= $locationSource;
+ if(defined("DEBUG") && DEBUG>=2) echo "YellowCommand::analyseStaticFiles detected url:$url<br/>\n";
+ }
+ }
+ ++$this->files;
+ } else {
+ $statusCode = 500;
+ echo "ERROR reading files: Can't read file '$fileName'!\n";
+ }
+ }
+ $this->links = count($links);
+ } else {
+ $statusCode = 500;
+ echo "ERROR reading files: Can't find files in directory '$path'!\n";
+ }
+ return array($statusCode, $links);
+ }
+
+ // Analyse links, detect status
+ function analyseLinks($path, $links)
+ {
+ $statusCode = 200;
+ $broken = $redirect = $data = array();
+ if(extension_loaded("curl"))
+ {
+ $staticUrl = $this->yellow->config->get("staticUrl");
+ list($scheme, $address, $base) = $this->yellow->lookup->getUrlInformation($staticUrl);
+ $staticLocations = $this->getContentLocations(true);
+ uksort($links, "strnatcasecmp");
+ foreach($links as $url=>$value)
+ {
+ if(defined("DEBUG") && DEBUG>=1) echo "YellowCommand::analyseLinks url:$url\n";
+ if(preg_match("#^$staticUrl#", $url))
+ {
+ $location = substru($url, 32);
+ $fileName = $path.substru($url, 32);
+ if(is_readable($fileName)) continue;
+ if(in_array($location, $staticLocations)) continue;
+ }
+ if(preg_match("/^(http|https):/", $url))
+ {
+ $referer = "$scheme://$address".(($pos = strposu($value, ',')) ? substru($value, 0, $pos) : $value);
+ $statusCodeUrl = $this->getLinkStatus($url, $referer);
+ if($statusCodeUrl!=200)
+ {
+ $statusCode = max($statusCode, $statusCodeUrl);
+ $data[$url] = "$statusCodeUrl,$value";
+ }
+ }
+ }
+ foreach($data as $url=>$value)
+ {
+ $locations = preg_split("/\s*,\s*/", $value);
+ $statusCodeUrl = array_shift($locations);
+ foreach($locations as $location)
+ {
+ if($statusCodeUrl>=300 && $statusCodeUrl<=399) {
+ $redirect["$scheme://$address$location -> $url - ".$this->getStatusFormatted($statusCodeUrl)] = $statusCodeUrl;
+ } else {
+ $broken["$scheme://$address$location -> $url - ".$this->getStatusFormatted($statusCodeUrl)] = $statusCodeUrl;
+ }
+ }
+ }
+ } else {
+ $statusCode = 500;
+ echo "ERROR checking links: Plugin 'command' requires cURL library!\n";
+ }
+ return array($statusCode, $broken, $redirect);
+ }
+
+ // Show links
+ function showLinks($data, $text)
+ {
+ if(!empty($data))
+ {
+ echo "$text\n\n";
+ uksort($data, "strnatcasecmp");
+ $data = array_slice($data, 0, 99);
+ foreach($data as $key=>$value) echo "- $key\n";
+ echo "\n";
+ }
+ }
// Clean static files
function cleanCommand($args)
@@ -240,7 +400,7 @@ class YellowCommand
list($command, $path, $location) = $args;
if(empty($location) || $location[0]=='/')
{
- $statusCode = $this->cleanStatic($path, $location);
+ $statusCode = $this->cleanStaticFiles($path, $location);
echo "Yellow $command: Static file".(empty($location) ? "s" : "")." ".($statusCode!=200 ? "not " : "")."cleaned\n";
} else {
$statusCode = 400;
@@ -250,7 +410,7 @@ class YellowCommand
}
// Clean static files and directories
- function cleanStatic($path, $location)
+ function cleanStaticFiles($path, $location)
{
$statusCode = 200;
$path = rtrim(empty($path) ? $this->yellow->config->get("staticDir") : $path, '/');
@@ -259,7 +419,13 @@ class YellowCommand
$statusCode = max($statusCode, $this->commandBroadcast("clean", "all"));
$statusCode = max($statusCode, $this->cleanStaticDirectory($path));
} else {
- $statusCode = $this->cleanStaticFile($path, $location);
+ if($this->yellow->lookup->isFileLocation($location))
+ {
+ $fileName = $this->getStaticFile($path, $location, $statusCode);
+ $statusCode = $this->cleanStaticFile($fileName);
+ } else {
+ $statusCode = $this->cleanStaticDirectory($path.$location);
+ }
}
return $statusCode;
}
@@ -280,10 +446,9 @@ class YellowCommand
}
// Clean static file
- function cleanStaticFile($path, $location)
+ function cleanStaticFile($fileName)
{
$statusCode = 200;
- $fileName = $this->getStaticFile($path, $location, $statusCode);
if(is_file($fileName))
{
if(!$this->yellow->toolbox->deleteFile($fileName))
@@ -365,6 +530,18 @@ class YellowCommand
return $fileName;
}
+ // Return static location
+ function getStaticLocation($path, $fileName)
+ {
+ $location = substru($fileName, strlenu($path));
+ if(basename($location)==$this->yellow->config->get("staticDefaultFile"))
+ {
+ $defaultFileLength = strlenu($this->yellow->config->get("staticDefaultFile"));
+ $location = substru($location, 0, -$defaultFileLength);
+ }
+ return $location;
+ }
+
// Return static redirect
function getStaticRedirect($location)
{
@@ -374,9 +551,15 @@ class YellowCommand
$output .= "</head>\n</html>";
return $output;
}
+
+ // Return human readable status
+ function getStatusFormatted($statusCode)
+ {
+ return $this->yellow->toolbox->getHttpStatusFormatted($statusCode, true);
+ }
// Return content locations
- function getContentLocations()
+ function getContentLocations($includeAll = false)
{
$locations = array();
$staticUrl = $this->yellow->config->get("staticUrl");
@@ -384,7 +567,7 @@ class YellowCommand
$this->yellow->page->setRequestInformation($scheme, $address, $base, "", "");
foreach($this->yellow->pages->index(true, true) as $page)
{
- if($page->get("status")!="ignore" && $page->get("status")!="draft")
+ if(($page->get("status")!="ignore" && $page->get("status")!="draft") || $includeAll)
{
array_push($locations, $page->location);
}
@@ -458,6 +641,27 @@ class YellowCommand
}
return array($statusCode, $data);
}
+
+ // Return link status
+ function getLinkStatus($url, $referer)
+ {
+ if(extension_loaded("curl"))
+ {
+ $curlHandle = curl_init();
+ curl_setopt($curlHandle, CURLOPT_URL, $url);
+ curl_setopt($curlHandle, CURLOPT_REFERER, $referer);
+ curl_setopt($curlHandle, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; DatenstromYellow/".YellowCore::VERSION."; LinkChecker)");
+ curl_setopt($curlHandle, CURLOPT_NOBODY, 1);
+ curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 30);
+ curl_exec($curlHandle);
+ $statusCode = curl_getinfo($curlHandle, CURLINFO_HTTP_CODE);
+ curl_close($curlHandle);
+ if(defined("DEBUG") && DEBUG>=2) echo "YellowCommand::getLinkStatus status:$statusCode url:$url<br/>\n";
+ } else {
+ $statusCode = 500;
+ }
+ return $statusCode;
+ }
}
$yellow->plugins->register("command", "YellowCommand", YellowCommand::VERSION);
diff --git a/system/plugins/update.php b/system/plugins/update.php
@@ -5,7 +5,7 @@
class YellowUpdate
{
- const VERSION = "0.7.9";
+ const VERSION = "0.7.10";
var $yellow; //access to API
var $updates; //number of updates
@@ -18,7 +18,6 @@ class YellowUpdate
$this->yellow->config->setDefault("updateInformationFile", "update.ini");
$this->yellow->config->setDefault("updateVersionFile", "version.ini");
$this->yellow->config->setDefault("updateResourceFile", "resource.ini");
- $this->yellow->config->setDefault("newFile", "page-new-(.*).txt"); //TODO: remove with next release
}
// Handle startup
@@ -826,7 +825,7 @@ class YellowUpdate
}
$curlHandle = curl_init();
curl_setopt($curlHandle, CURLOPT_URL, $urlRequest);
- curl_setopt($curlHandle, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; YellowCore/".YellowCore::VERSION).")";
+ curl_setopt($curlHandle, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; DatenstromYellow/".YellowCore::VERSION."; SoftwareUpdater)");
curl_setopt($curlHandle, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 30);
$rawData = curl_exec($curlHandle);
