• R/O
  • HTTP
  • SSH
  • HTTPS

Commit

Tags
No Tags

Frequently used words (click to add to your profile)

javac++androidlinuxc#objective-cqtwindows誰得cocoapythonphprubygameguibathyscaphec翻訳omegat計画中(planning stage)frameworktwittertestdomvb.netdirectxbtronarduinopreviewerゲームエンジン

Commit MetaInfo

Revisiond11e12ee8df59ce6323cc1ad1b8c4495b294afab (tree)
Time2018-02-27 00:10:34
Authorumorigu <umorigu@gmai...>
Commiterumorigu

Log Message

BugTrack/560 tracker_list speeding up with cache

* Implement 2-layer caching

  • Converted HTML cache
  • Contents cache for list pages

* Get only updated/deleted page contents
* Detect page updates by file or page timestamps
* Refer cache/recent.dat file to detect updated pages
* Refer RecentDeleted page to detect deleted pages
* Handle page links properly on tracker_list table

Change Summary

Incremental Difference

--- a/plugin/tracker.inc.php
+++ b/plugin/tracker.inc.php
@@ -1,7 +1,7 @@
11 <?php
22 // PukiWiki - Yet another WikiWikiWeb clone
33 // tracker.inc.php
4-// Copyright 2003-2017 PukiWiki Development Team
4+// Copyright 2003-2018 PukiWiki Development Team
55 // License: GPL v2 or (at your option) any later version
66 //
77 // Issue tracker plugin (See Also bugtrack plugin)
@@ -15,6 +15,9 @@ define('TRACKER_LIST_EXCLUDE_PATTERN','#^SubMenu$|/#');
1515 // 項目の取り出しに失敗したページを一覧に表示する
1616 define('TRACKER_LIST_SHOW_ERROR_PAGE',TRUE);
1717
18+// Use cache
19+define('TRACKER_LIST_USE_CACHE', TRUE);
20+
1821 function plugin_tracker_convert()
1922 {
2023 global $vars;
@@ -684,12 +687,15 @@ function plugin_tracker_list_action()
684687 }
685688 function plugin_tracker_getlist($page,$refer,$config_name,$list,$order='',$limit=NULL)
686689 {
687- $config = new Config('plugin/tracker/'.$config_name);
690+ global $whatsdeleted;
688691
692+ $config = new Config('plugin/tracker/'.$config_name);
689693 if (!$config->read())
690694 {
691695 return "<p>config file '".htmlsc($config_name)."' is not exist.</p>";
692696 }
697+ //pkwk_log('AAAA');
698+ //pkwk_log(print_r(['page:' => $page, 'refer' => $refer, 'config_name' => $config_name, 'list' => $list, 'order' => $order], true));
693699
694700 $config->config_name = $config_name;
695701
@@ -698,9 +704,79 @@ function plugin_tracker_getlist($page,$refer,$config_name,$list,$order='',$limit
698704 return "<p>config file '".make_pagelink($config->page.'/'.$list)."' not found.</p>";
699705 }
700706
701- $list = new Tracker_list($page,$refer,$config,$list);
702- $list->sort($order);
703- return $list->toString($limit);
707+ $cache_enabled = defined('TRACKER_LIST_USE_CACHE') && TRACKER_LIST_USE_CACHE &&
708+ defined('JSON_UNESCAPED_UNICODE') && defined('PKWK_UTF8_ENABLE');
709+ $cache_filepath = CACHE_DIR . encode($page) . '.tracker';
710+ $cachedata = null;
711+ $cache_format_version = 1;
712+ if ($cache_enabled && file_exists($cache_filepath)) {
713+ $config_filetime = get_filetime($config->page);
714+ $config_list_filetime = get_filetime($config->page.'/'. $list);
715+ $json_cached = pkwk_file_get_contents($cache_filepath);
716+ if ($json_cached) {
717+ $wrapdata = json_decode($json_cached, true);
718+ if (is_array($wrapdata)) {
719+ if (isset($wrapdata['version'], $wrapdata['html'], $wrapdata['refreshed_at'])) {
720+ $cache_time_prev = $wrapdata['refreshed_at'];
721+ if ($cache_format_version === $wrapdata['version']) {
722+ if ($config_filetime === $wrapdata['config_updated_at'] &&
723+ $config_list_filetime === $wrapdata['config_list_updated_at']) {
724+ $cachedata = $wrapdata;
725+ } else {
726+ // (Ignore) delete file
727+ unlink($cache_filepath);
728+ }
729+ }
730+ }
731+ }
732+ }
733+ }
734+ // Check recent.dat timestamp
735+ $recent_dat_filemtime = filemtime(CACHE_DIR . PKWK_MAXSHOW_CACHE);
736+ // Check RecentDeleted timestamp
737+ $recent_deleted_filetime = get_filetime($whatsdeleted);
738+ if (is_null($cachedata)) {
739+ $cachedata = array();
740+ } else {
741+ if ($recent_dat_filemtile !== false) {
742+ if ($recent_dat_filemtime === $cachedata['recent_dat_filemtime'] &&
743+ $recent_deleted_filetime === $cachedata['recent_deleted_filetime'] &&
744+ $order === $cachedata['order']) {
745+ // recent.dat is unchanged
746+ // RecentDeleted is unchanged
747+ // order is unchanged
748+ return $cachedata['html'];
749+ }
750+ }
751+ }
752+ $cache_holder = $cachedata;
753+ $tracker_list = new Tracker_list($page,$refer,$config,$list,$cache_holder);
754+ if ($order === $cache_holder['order'] &&
755+ empty($tracker_list->newly_deleted_pages) &&
756+ empty($tracker_list->newly_updated_pages) &&
757+ !$tracker_list->link_update_required) {
758+ $result = $cache_holder['html'];
759+ } else {
760+ $tracker_list->sort($order);
761+ $result = $tracker_list->toString($limit);
762+ }
763+ if ($cache_enabled) {
764+ $refreshed_at = time();
765+ $json = array(
766+ 'refreshed_at' => $refreshed_at,
767+ 'rows' => $tracker_list->rows,
768+ 'html' => $result,
769+ 'order' => $order,
770+ 'config_updated_at' => $config_filetime,
771+ 'config_list_updated_at' => $config_list_filetime,
772+ 'recent_dat_filemtime' => $recent_dat_filemtime,
773+ 'recent_deleted_filetime' => $recent_deleted_filetime,
774+ 'link_pages' => $tracker_list->link_pages,
775+ 'version' => $cache_format_version);
776+ $cache_body = json_encode($json, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
777+ file_put_contents($cache_filepath, $cache_body, LOCK_EX);
778+ }
779+ return $result;
704780 }
705781
706782 // 一覧クラス
@@ -715,13 +791,16 @@ class Tracker_list
715791 var $rows;
716792 var $order;
717793 var $sort_keys;
794+ var $newly_deleted_pages = array();
795+ var $newly_updated_pages = array();
718796
719- function Tracker_list($page,$refer,&$config,$list)
797+ function Tracker_list($page,$refer,&$config,$list,&$cache_holder)
720798 {
721- $this->__construct($page, $refer, $config, $list);
799+ $this->__construct($page, $refer, $config, $list, $cache_holder);
722800 }
723- function __construct($page,$refer,&$config,$list)
801+ function __construct($page,$refer,&$config,$list,&$cache_holder)
724802 {
803+ global $whatsdeleted;
725804 $this->page = $page;
726805 $this->config = &$config;
727806 $this->list = $list;
@@ -746,22 +825,190 @@ class Tracker_list
746825 $this->pattern .= '(.*?)';
747826 }
748827 }
749- // ページの列挙と取り込み
750- $this->rows = array();
751- $pattern = "$page/";
752- $pattern_len = strlen($pattern);
753- foreach (get_existpages() as $_page)
754- {
755- if (strpos($_page,$pattern) === 0)
828+ if (empty($cache_holder)) {
829+ // List pages and get contents (non-cache behavior)
830+ $this->rows = array();
831+ $pattern = "$page/";
832+ $pattern_len = strlen($pattern);
833+ foreach (get_existpages() as $_page)
756834 {
757- $name = substr($_page,$pattern_len);
758- if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN,$name))
835+ if (substr($_page, 0, $pattern_len) === $pattern)
759836 {
760- continue;
837+ $name = substr($_page,$pattern_len);
838+ if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN,$name))
839+ {
840+ continue;
841+ }
842+ $this->add($_page,$name);
843+ }
844+ }
845+ $this->link_pages = $this->get_filetimes($this->get_all_links());
846+ } else {
847+ // Cache-available behavior
848+ // Check RecentDeleted timestamp
849+ $cached_rows = $this->decode_cached_rows($cache_holder['rows']);
850+ $updated_linked_pages = array();
851+ $newly_deleted_pages = array();
852+ $pattern = "$page/";
853+ $pattern_len = strlen($pattern);
854+ $recent_deleted_filetime = get_filetime($whatsdeleted);
855+ $deleted_page_list = array();
856+ if ($recent_deleted_filetime !== $cache_holder['recent_deleted_filetime']) {
857+ //pkwk_log(print_r());
858+ foreach (plugin_tracker_get_source($whatsdeleted) as $line) {
859+ $m = null;
860+ if (preg_match('#\[\[([^\]]+)\]\]#', $line, $m)) {
861+ $_page = $m[1];
862+ if (is_pagename($_page)) {
863+ $deleted_page_list[] = $m[1];
864+ }
865+ }
866+ }
867+ foreach ($deleted_page_list as $_page) {
868+ if (substr($_page, 0, $pattern_len) === $pattern) {
869+ $name = substr($_page, $pattern_len);
870+ if (!is_page($_page) && isset($cached_rows[$name]) &&
871+ !preg_match(TRACKER_LIST_EXCLUDE_PATTERN, $name)) {
872+ // This page was just deleted
873+ array_push($newly_deleted_pages, $_page);
874+ unset($cached_rows[$name]);
875+ }
876+ }
761877 }
762- $this->add($_page,$name);
763878 }
879+ $this->newly_deleted_pages = $newly_deleted_pages;
880+ $updated_pages = array();
881+ $this->rows = $cached_rows;
882+ // Check recent.dat timestamp
883+ $recent_dat_filemtime = filemtime(CACHE_DIR . PKWK_MAXSHOW_CACHE);
884+ $updated_page_list = array();
885+ if ($recent_dat_filemtime !== $cache_holder['recent_dat_filemtime']) {
886+ // recent.dat was updated. Search which page was updated.
887+ $target_pages = array();
888+ // Active page file time (1 hour before timestamp of recent.dat)
889+ $target_filetime = $cache_holder['recent_dat_filemtime'] - LOCALZONE - 60 * 60;
890+ foreach (get_recent_files() as $_page=>$time) {
891+ if ($time <= $target_filetime) {
892+ // Older updated pages
893+ break;
894+ }
895+ $updated_page_list[$_page] = $time;
896+ $name = substr($_page, $pattern_len);
897+ if (substr($_page, 0, $pattern_len) === $pattern) {
898+ $name = substr($_page, $pattern_len);
899+ if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN, $name)) {
900+ continue;
901+ }
902+ // Tracker target page
903+ if (isset($this->rows[$name])) {
904+ // Existing page
905+ $row = $this->rows[$name];
906+ if ($row['_update'] === get_filetime($_page)) {
907+ // Same as cache
908+ continue;
909+ } else {
910+ // Found updated page
911+ $updated_pages[] = $_page;
912+ unset($this->rows[$name]);
913+ $this->add($_page, $name);
914+ }
915+ } else {
916+ // Add new page
917+ $updated_pages[] = $_page;
918+ $this->add($_page, $name);
919+ }
920+ }
921+ }
922+ }
923+ $this->newly_updated_pages = $updated_pages;
924+ $new_link_names = $this->get_all_links();
925+ $old_link_map = array();
926+ foreach ($cache_holder['link_pages'] as $link_page) {
927+ $old_link_map[$link_page['page']] = $link_page['filetime'];
928+ }
929+ $new_link_map = $old_link_map;
930+ $link_update_required = false;
931+ foreach ($deleted_page_list as $_page) {
932+ if (in_array($_page, $new_link_names)) {
933+ if (isset($old_link_map[$_page])) {
934+ // This link keeps existing
935+ if (!is_page($_page)) {
936+ // OK. Confirmed the page doesn't exist
937+ if ($old_link_map[$_page] === 0) {
938+ // Do nothing (From no-page to no-page)
939+ } else {
940+ // This page was just deleted
941+ $new_link_map[$_page] = get_filetime($_page);
942+ $link_update_required = true;
943+ }
944+ }
945+ } else {
946+ // This link was just added
947+ $new_link_map[$_page] = get_filetime($_page);
948+ $link_update_required = true;
949+ }
950+ }
951+ }
952+ foreach ($updated_page_list as $_page=>$time) {
953+ if (in_array($_page, $new_link_names)) {
954+ if (isset($old_link_map[$_page])) {
955+ // This link keeps existing
956+ if (is_page($_page)) {
957+ // OK. Confirmed the page now exists
958+ if ($old_link_map[$_page] === 0) {
959+ // This page was just added
960+ $new_link_map[$_page] = get_filetime($_page);
961+ $link_update_required = true;
962+ } else {
963+ // Do nothing (existing-page to existing-page)
964+ }
965+ }
966+ } else {
967+ // This link was just added
968+ $new_link_map[$_page] = get_filetime($_page);
969+ $link_update_required = true;
970+ }
971+ }
972+ }
973+ $new_link_pages = array();
974+ foreach ($new_link_map as $_page => $time) {
975+ $new_link_pages[] = array(
976+ 'page' => $_page,
977+ 'filetime' => $time,
978+ );
979+ }
980+ $this->link_pages = $new_link_pages;
981+ $this->link_update_required = $link_update_required;
982+ }
983+ }
984+ function decode_cached_rows($decoded_rows)
985+ {
986+ $ar = array();
987+ foreach ($decoded_rows as $row) {
988+ $ar[$row['_real']] = $row;
989+ }
990+ return $ar;
991+ }
992+ function get_all_links() {
993+ $ar = array();
994+ foreach ($this->rows as $row) {
995+ foreach ($row['_links'] as $link) {
996+ $ar[$link] = 0;
997+ }
998+ }
999+ pkwk_log('LINK');
1000+ pkwk_log(print_r(array_keys($ar), true));
1001+ return array_keys($ar);
1002+ }
1003+ function get_filetimes($pages) {
1004+ $filetimes = array();
1005+ foreach ($pages as $page) {
1006+ $filetimes[] = array(
1007+ 'page' => $page,
1008+ 'filetime' => get_filetime($page),
1009+ );
7641010 }
1011+ return $filetimes;
7651012 }
7661013 function add($page,$name)
7671014 {
@@ -786,22 +1033,38 @@ class Tracker_list
7861033 }
7871034 $source = join('',preg_replace('/^(\*{1,3}.*)\[#[A-Za-z][\w-]+\](.*)$/','$1$2',$source));
7881035
789- // デフォルト値
790- $this->rows[$name] = array(
1036+ // Default value
1037+ $page_filetime = get_filetime($page);
1038+ $row = array(
7911039 '_page' => "[[$page]]",
7921040 '_refer' => $this->page,
7931041 '_real' => $name,
794- '_update'=> get_filetime($page),
795- '_past' => get_filetime($page)
1042+ '_update'=> $page_filetime,
1043+ '_past' => $page_filetime,
7961044 );
797- if ($this->rows[$name]['_match'] = preg_match("/{$this->pattern}/s",$source,$matches))
1045+ $links = array();
1046+ if ($row['_match'] = preg_match("/{$this->pattern}/s",$source,$matches))
7981047 {
7991048 array_shift($matches);
8001049 foreach ($this->pattern_fields as $key=>$field)
8011050 {
802- $this->rows[$name][$field] = trim($matches[$key]);
1051+ $row[$field] = trim($matches[$key]);
1052+ if ($field === '_refer') {
1053+ continue;
1054+ }
1055+ $lmatch = null;
1056+ if (preg_match('/\[\[([^\]\]]+)\]/', $row[$field], $lmatch)) {
1057+ $link = $lmatch[1];
1058+ if (is_pagename($link) && $link !== $this->page && $link !== $page) {
1059+ if (!in_array($link, $links)) {
1060+ $links[] = $link;
1061+ }
1062+ }
1063+ }
8031064 }
8041065 }
1066+ $row['_links'] = $links;
1067+ $this->rows[$name] = $row;
8051068 }
8061069 function compare($a, $b)
8071070 {
--- a/plugin/tracker_list.inc.php
+++ b/plugin/tracker_list.inc.php
@@ -1,7 +1,7 @@
11 <?php
22 // PukiWiki - Yet another WikiWikiWeb clone
33 // tracker_list.inc.php
4-// Copyright 2003-2017 PukiWiki Development Team
4+// Copyright 2003-2018 PukiWiki Development Team
55 // License: GPL v2 or (at your option) any later version
66 //
77 // Issue tracker list plugin (a part of tracker plugin)