From 2af15f2136dc4926157852c19629285b6a7b1dd8 Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Thu, 6 Feb 2020 17:46:32 +1100 Subject: [PATCH 1/9] Increase timeout for cases when caches have been just purged #179. --- classes/local/outagelib.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/local/outagelib.php b/classes/local/outagelib.php index cf8ebbd..8a8b496 100644 --- a/classes/local/outagelib.php +++ b/classes/local/outagelib.php @@ -64,7 +64,7 @@ class outagelib { curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, false); curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false); curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 5); // It is localhost, time to connect is enough. - curl_setopt($curl, CURLOPT_TIMEOUT, 15); // It is localhost, time to fetch index is enough. + curl_setopt($curl, CURLOPT_TIMEOUT, 60); $contents = curl_exec($curl); $mime = curl_getinfo($curl, CURLINFO_CONTENT_TYPE); curl_close($curl); From ae85c9436449ccb84c49292ac67523805f6708bf Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Thu, 6 Feb 2020 18:01:39 +1100 Subject: [PATCH 2/9] Cook file url properly when Moodle is installed in a sub folder #179. --- .../maintenance_static_page_generator.php | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/classes/local/controllers/maintenance_static_page_generator.php b/classes/local/controllers/maintenance_static_page_generator.php index e9e20e3..5164152 100644 --- a/classes/local/controllers/maintenance_static_page_generator.php +++ b/classes/local/controllers/maintenance_static_page_generator.php @@ -156,7 +156,25 @@ class maintenance_static_page_generator { if (maintenance_static_page_io::is_url($originalurl)) { $fullurl = $originalurl; } else if ($originalurl[0] == '/') { - $fullurl = $CFG->wwwroot.$originalurl; + if (strpos($CFG->wwwroot, 'http://') === 0) { + $domain = substr($CFG->wwwroot, 7); + if (strpos($domain, '/') > 0) { + $base = substr($domain, 0, strpos($domain, '/')); + } else { + $base = $domain; + } + $fullurl = 'http://'.$base.$originalurl; + } else if (strpos($CFG->wwwroot, 'https://') === 0) { + $domain = substr($CFG->wwwroot, 8); + if (strpos($domain, '/') > 0) { + $base = substr($domain, 0, strpos($domain, '/')); + } else { + $base = $domain; + } + $fullurl = 'https://'.$base.$originalurl; + } else { + $fullurl = $CFG->wwwroot.$originalurl; + } } else { $fullurl = $baseref.'/'.$originalurl; } From 7a1fa1f91b60bd5dc86175d73c315bf9fad05129 Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Thu, 6 Feb 2020 18:02:59 +1100 Subject: [PATCH 3/9] Tweak regexp and exclude non-url entries #179. --- classes/local/controllers/maintenance_static_page_generator.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/local/controllers/maintenance_static_page_generator.php b/classes/local/controllers/maintenance_static_page_generator.php index 5164152..5cf0eba 100644 --- a/classes/local/controllers/maintenance_static_page_generator.php +++ b/classes/local/controllers/maintenance_static_page_generator.php @@ -148,7 +148,7 @@ class maintenance_static_page_generator { global $CFG; $contents = file_get_contents($filename); - if (!preg_match_all('#url\([\'"]?([^\'"\)]+)#', $contents, $matches)) { + if (!preg_match_all('#url\([\'"]?(?!data:)([^\'"\)]+)#', $contents, $matches)) { return; } foreach ($matches[1] as $originalurl) { From a361ae4008b72287ca2d6eb6a83b33069d9816e8 Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Fri, 7 Feb 2020 14:25:33 +1100 Subject: [PATCH 4/9] Simplify url building for update_link_stylesheet_parse() #179. --- .../maintenance_static_page_generator.php | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/classes/local/controllers/maintenance_static_page_generator.php b/classes/local/controllers/maintenance_static_page_generator.php index 5cf0eba..6de8c25 100644 --- a/classes/local/controllers/maintenance_static_page_generator.php +++ b/classes/local/controllers/maintenance_static_page_generator.php @@ -156,25 +156,8 @@ class maintenance_static_page_generator { if (maintenance_static_page_io::is_url($originalurl)) { $fullurl = $originalurl; } else if ($originalurl[0] == '/') { - if (strpos($CFG->wwwroot, 'http://') === 0) { - $domain = substr($CFG->wwwroot, 7); - if (strpos($domain, '/') > 0) { - $base = substr($domain, 0, strpos($domain, '/')); - } else { - $base = $domain; - } - $fullurl = 'http://'.$base.$originalurl; - } else if (strpos($CFG->wwwroot, 'https://') === 0) { - $domain = substr($CFG->wwwroot, 8); - if (strpos($domain, '/') > 0) { - $base = substr($domain, 0, strpos($domain, '/')); - } else { - $base = $domain; - } - $fullurl = 'https://'.$base.$originalurl; - } else { - $fullurl = $CFG->wwwroot.$originalurl; - } + $rooturl = parse_url($CFG->wwwroot); + $fullurl = $rooturl['scheme'].'://'.$rooturl['host'].$originalurl; } else { $fullurl = $baseref.'/'.$originalurl; } From 26baaf9993efd544c811e21827bb80fb5c3897aa Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Fri, 7 Feb 2020 15:41:52 +1100 Subject: [PATCH 5/9] Fix unit tests #179. --- tests/phpunit/local/controllers/fixtures/withurls-quoted.css | 2 +- tests/phpunit/local/controllers/fixtures/withurls.css | 2 +- .../local/controllers/maintenance_static_page_test.php | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/phpunit/local/controllers/fixtures/withurls-quoted.css b/tests/phpunit/local/controllers/fixtures/withurls-quoted.css index 012773c..884fef7 100644 --- a/tests/phpunit/local/controllers/fixtures/withurls-quoted.css +++ b/tests/phpunit/local/controllers/fixtures/withurls-quoted.css @@ -3,5 +3,5 @@ a { } div { - background-image: url('/auth/outage/tests/phpunit/local/controllers/fixtures/catalyst.png'); + background-image: url('/moodle/auth/outage/tests/phpunit/local/controllers/fixtures/catalyst.png'); } \ No newline at end of file diff --git a/tests/phpunit/local/controllers/fixtures/withurls.css b/tests/phpunit/local/controllers/fixtures/withurls.css index 7e1e985..c5b57fa 100644 --- a/tests/phpunit/local/controllers/fixtures/withurls.css +++ b/tests/phpunit/local/controllers/fixtures/withurls.css @@ -3,5 +3,5 @@ a { } div { - background-image: url(/auth/outage/tests/phpunit/local/controllers/fixtures/catalyst.png); + background-image: url(/moodle/auth/outage/tests/phpunit/local/controllers/fixtures/catalyst.png); } \ No newline at end of file diff --git a/tests/phpunit/local/controllers/maintenance_static_page_test.php b/tests/phpunit/local/controllers/maintenance_static_page_test.php index 4176ee5..29a6805 100644 --- a/tests/phpunit/local/controllers/maintenance_static_page_test.php +++ b/tests/phpunit/local/controllers/maintenance_static_page_test.php @@ -101,7 +101,7 @@ class maintenance_static_page_test extends auth_outage_base_testcase { $page->generate(); // Check for css file. - self::assertFileExists($page->get_io()->get_resources_folder().'/622ef6e83acfcb274cdf37bdb3bffa0923f9a7ad.dGV4dC9wbGFpbg'); + self::assertFileExists($page->get_io()->get_resources_folder().'/d8643101d96b093e642b15544e4d1f7815b5ba55.dGV4dC9wbGFpbg'); // Check for catalyst.png file referenced in url(..) of css. self::assertFileExists($page->get_io()->get_resources_folder().'/ff7f7f87a26a908fc72930eaefb6b57306361d16.aW1hZ2UvcG5n'); @@ -116,7 +116,7 @@ class maintenance_static_page_test extends auth_outage_base_testcase { $page->generate(); // Check for css file. - self::assertFileExists($page->get_io()->get_resources_folder().'/1d84b6d321fef780237f84834b7316c079221a31.dGV4dC9wbGFpbg'); + self::assertFileExists($page->get_io()->get_resources_folder().'/9fe2374b03953e1949d54ab750be2d8706891c03.dGV4dC9wbGFpbg'); // Check for catalyst.png file referenced in url(..) of css. self::assertFileExists($page->get_io()->get_resources_folder().'/ff7f7f87a26a908fc72930eaefb6b57306361d16.aW1hZ2UvcG5n'); From 7ad82886d718a06a849a97962298797ab3b86d4b Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Mon, 10 Feb 2020 11:12:14 +1100 Subject: [PATCH 6/9] Move regexps to a separate method #179. --- .../maintenance_static_page_generator.php | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/classes/local/controllers/maintenance_static_page_generator.php b/classes/local/controllers/maintenance_static_page_generator.php index 6de8c25..d58e87b 100644 --- a/classes/local/controllers/maintenance_static_page_generator.php +++ b/classes/local/controllers/maintenance_static_page_generator.php @@ -138,6 +138,17 @@ class maintenance_static_page_generator { } } + /** + * Retrieves all URLs from file content using regular expressions. + * + * @param string $contents Content of the file + * @return array Array of all matches in multi-dimensional array + */ + public function get_urls_from_stylesheet($contents) { + preg_match_all('#url\([\'"]?(?!data:)([^\'"\)]+)#', $contents, $matches); + return $matches; + } + /** * Checks for urls inside filename. * @@ -148,9 +159,8 @@ class maintenance_static_page_generator { global $CFG; $contents = file_get_contents($filename); - if (!preg_match_all('#url\([\'"]?(?!data:)([^\'"\)]+)#', $contents, $matches)) { - return; - } + $matches = $this->get_urls_from_stylesheet($contents); + foreach ($matches[1] as $originalurl) { // Allow incomplete URLs in CSS, assume it is from moodle root. if (maintenance_static_page_io::is_url($originalurl)) { From aadd7eaeeecfafcaaec26706d6a294e4d52f4600 Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Mon, 10 Feb 2020 12:17:05 +1100 Subject: [PATCH 7/9] Cover get_urls_from_stylesheet() method by unit tests #179. --- .../maintenance_static_page_test.php | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/phpunit/local/controllers/maintenance_static_page_test.php b/tests/phpunit/local/controllers/maintenance_static_page_test.php index 29a6805..dfd6ac5 100644 --- a/tests/phpunit/local/controllers/maintenance_static_page_test.php +++ b/tests/phpunit/local/controllers/maintenance_static_page_test.php @@ -25,6 +25,7 @@ use auth_outage\local\controllers\maintenance_static_page; use auth_outage\local\controllers\maintenance_static_page_io; +use auth_outage\local\controllers\maintenance_static_page_generator; use auth_outage\task\update_static_page; defined('MOODLE_INTERNAL') || die(); @@ -395,4 +396,65 @@ class maintenance_static_page_test extends auth_outage_base_testcase { self::assertContains('', $generated); } + + /** + * Data provider for test_get_urls_from_stylesheet + * @return array + */ + public function test_get_urls_from_stylesheet_provider() { + $provider = array(); + + $filecontent = ""; + $provider[] = array($filecontent, 0); + + $filecontent = "background:url(/theme/image.php/_s/boost/core/1581292565/t/expanded)"; + $provider[] = array($filecontent, 1); + + $filecontent = "background:url('/theme/image.php/_s/boost/core/1581292565/t/expanded')"; + $provider[] = array($filecontent, 1); + + $filecontent = "src:url(\"/theme/font.php/boost/core/1581292565/fontawesome-webfont.eot?#iefix&v=4.7.0\")"; + $provider[] = array($filecontent, 1); + + $filecontent = "background-image:url(pix/vline-rtl.gif)"; + $provider[] = array($filecontent, 1); + + $filecontent = "background-image:url(data:image/gif;base64,R0lGODlhYADIAP=)"; + $provider[] = array($filecontent, 0); + + $filecontent = "background-image:url('data:image/gif;base64,R0lGODlhYADIAP=')"; + $provider[] = array($filecontent, 0); + + $filecontent = "background-image:url(\"data:image/svg+xml;charset=utf8,%3Csvg xmlns=\'http://www.w3.org/2000/svg\'". + "viewBox=\'0 0 8 8\'%3E%3Cpath fill=\'%23fff\' d=\'M6.564.75l-3.59 2.193z\'/%3E%3C/svg%3E\")"; + $provider[] = array($filecontent, 0); + + $filecontent = "background-image:url(pix/vline-rtl.gif)". + "background:url(/theme/image.php/_s/boost/core/1581292565/t/expanded)"; + $provider[] = array($filecontent, 2); + + $filecontent = "background-image:url(data:image/gif;base64,R0lGODlhYADIAP=)". + "src:url(\"/theme/font.php/boost/core/1581292565/fontawesome-webfont.eot?#iefix&v=4.7.0\")"; + $provider[] = array($filecontent, 1); + + return $provider; + } + + /** + * Tests get_urls_from_stylesheet() method to get all appropriate URLS from the file. + * + * @dataProvider test_get_urls_from_stylesheet_provider + * @param string $filecontent Content of the file + * @param int $count Expected quantity of found URLs + * @throws coding_exception + */ + public function test_get_urls_from_stylesheet($filecontent, $count) { + $this->resetAfterTest(true); + $generator = new maintenance_static_page_generator(new DOMDocument(), new maintenance_static_page_io()); + $matches = $generator->get_urls_from_stylesheet($filecontent); + + self::assertIsArray($matches); + self::assertCount(2, $matches); + self::assertCount($count, $matches[1]); + } } From b6f5db001ad7345a5147908511aa0ef209b22f73 Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Mon, 10 Feb 2020 12:53:52 +1100 Subject: [PATCH 8/9] Fix unit tests to support older versions of phpunit #179. --- .../phpunit/local/controllers/maintenance_static_page_test.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/phpunit/local/controllers/maintenance_static_page_test.php b/tests/phpunit/local/controllers/maintenance_static_page_test.php index dfd6ac5..b2238cc 100644 --- a/tests/phpunit/local/controllers/maintenance_static_page_test.php +++ b/tests/phpunit/local/controllers/maintenance_static_page_test.php @@ -453,7 +453,7 @@ class maintenance_static_page_test extends auth_outage_base_testcase { $generator = new maintenance_static_page_generator(new DOMDocument(), new maintenance_static_page_io()); $matches = $generator->get_urls_from_stylesheet($filecontent); - self::assertIsArray($matches); + self::assertInternalType('array', $matches); self::assertCount(2, $matches); self::assertCount($count, $matches[1]); } From 2cecb87317ec6ad888d85cd4f89210276514d437 Mon Sep 17 00:00:00 2001 From: Mikhail Golenkov Date: Mon, 10 Feb 2020 14:28:48 +1100 Subject: [PATCH 9/9] Format test_get_urls_from_stylesheet_provider() #179. --- .../maintenance_static_page_test.php | 52 ++++++------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/tests/phpunit/local/controllers/maintenance_static_page_test.php b/tests/phpunit/local/controllers/maintenance_static_page_test.php index b2238cc..f632284 100644 --- a/tests/phpunit/local/controllers/maintenance_static_page_test.php +++ b/tests/phpunit/local/controllers/maintenance_static_page_test.php @@ -402,42 +402,22 @@ class maintenance_static_page_test extends auth_outage_base_testcase { * @return array */ public function test_get_urls_from_stylesheet_provider() { - $provider = array(); - - $filecontent = ""; - $provider[] = array($filecontent, 0); - - $filecontent = "background:url(/theme/image.php/_s/boost/core/1581292565/t/expanded)"; - $provider[] = array($filecontent, 1); - - $filecontent = "background:url('/theme/image.php/_s/boost/core/1581292565/t/expanded')"; - $provider[] = array($filecontent, 1); - - $filecontent = "src:url(\"/theme/font.php/boost/core/1581292565/fontawesome-webfont.eot?#iefix&v=4.7.0\")"; - $provider[] = array($filecontent, 1); - - $filecontent = "background-image:url(pix/vline-rtl.gif)"; - $provider[] = array($filecontent, 1); - - $filecontent = "background-image:url(data:image/gif;base64,R0lGODlhYADIAP=)"; - $provider[] = array($filecontent, 0); - - $filecontent = "background-image:url('data:image/gif;base64,R0lGODlhYADIAP=')"; - $provider[] = array($filecontent, 0); - - $filecontent = "background-image:url(\"data:image/svg+xml;charset=utf8,%3Csvg xmlns=\'http://www.w3.org/2000/svg\'". - "viewBox=\'0 0 8 8\'%3E%3Cpath fill=\'%23fff\' d=\'M6.564.75l-3.59 2.193z\'/%3E%3C/svg%3E\")"; - $provider[] = array($filecontent, 0); - - $filecontent = "background-image:url(pix/vline-rtl.gif)". - "background:url(/theme/image.php/_s/boost/core/1581292565/t/expanded)"; - $provider[] = array($filecontent, 2); - - $filecontent = "background-image:url(data:image/gif;base64,R0lGODlhYADIAP=)". - "src:url(\"/theme/font.php/boost/core/1581292565/fontawesome-webfont.eot?#iefix&v=4.7.0\")"; - $provider[] = array($filecontent, 1); - - return $provider; + return [ + // Empty string. + ["", 0], + // URLs that should be retrieved. + ["background:url(/theme/image.php/_s/boost/core/1581292565/t/expanded)", 1], + ["background:url('/theme/image.php/_s/boost/core/1581292565/t/expanded')", 1], + ["src:url(\"/theme/font.php/boost/core/1581292565/fontawesome-webfont.eot?#iefix&v=4.7.0\")", 1], + ["background-image:url(pix/vline-rtl.gif)", 1], + // URLs that should not be retrieved. + ["background-image:url(data:image/gif;base64,R0lGODlhYADIAP=)", 0], + ["background-image:url('data:image/gif;base64,R0lGODlhYADIAP=')", 0], + ["background-image:url(\"data:image/svg+xml;charset=utf8,%3Csvg xmlns=\'http://www.w3.org/2000/svg\'\")", 0], + // Combination of URLs used above. + ["background-image:url(pix/vline-rtl.gif) background:url(/theme/image.php/_s/boost/core/158/t/expanded)", 2], + ["background-image:url(data:image/gif;base64,R0lG=)src:url(\"/theme/font.php/fontawesome-webfont.eot\")", 1], + ]; } /**