From 32fca654cb68f2b54b5c054a6b373d98e0e5daca Mon Sep 17 00:00:00 2001 From: Wytze van der Raay Date: Wed, 20 Nov 2013 16:05:13 +0000 Subject: [PATCH] Fix for https://bugs.cacert.org/view.php?id=1004 "performance of CAcert webserver is hampered by simultaneous stats.php execution" --- scripts/cron/refresh_stats.php | 307 +++++++++++++++++++++++ www/stats.php | 428 ++++++++++++--------------------- 2 files changed, 467 insertions(+), 268 deletions(-) create mode 100755 scripts/cron/refresh_stats.php diff --git a/scripts/cron/refresh_stats.php b/scripts/cron/refresh_stats.php new file mode 100755 index 0000000..2a3d2b5 --- /dev/null +++ b/scripts/cron/refresh_stats.php @@ -0,0 +1,307 @@ +#!/usr/bin/php -q + NOW()"); + $certs += tc("select count(*) as `count` from `emailcerts` + where `revoked` = 0 and `expire` > NOW()"); + $certs += tc("select count(*) as `count` from `gpg` + where `expire` > NOW()"); + $certs += tc("select count(*) as `count` from `orgdomaincerts` + where `revoked` = 0 and `expire` > NOW()"); + $certs += tc("select count(*) as `count` from `orgemailcerts` + where `revoked` = 0 and `expire` > NOW()"); + $stats['valid_certificates'] = number_format($certs); + + $stats['assurances_made'] = number_format(tc( + "select count(*) as `count` from `notary` + where `method` = '' or `method` = 'Face to Face Meeting'")); + + $stats['users_1to49'] = number_format(tc( + "select count(*) as `count` from ( + select 1 from `notary` + where `deleted` = 0 + group by `to` + having sum(`points`) > 0 and sum(`points`) < 50 + ) as `low_points`")); + + $stats['users_50to99'] = number_format(tc( + "select count(*) as `count` from ( + select 1 from `notary` + where `deleted` = 0 + group by `to` + having sum(`points`) >= 50 and sum(`points`) < 100 + ) as `high_points`")); + + $stats['assurer_candidates'] = number_format(tc( + "select count(*) as `count` from `users` + where ( + select sum(`points`) from `notary` + where `to`=`users`.`id` + and `deleted` = 0 + ) >= 100 + and not exists( + select 1 from `cats_passed` as `cp`, `cats_variant` as `cv` + where `cp`.`user_id`=`users`.`id` + and `cp`.`variant_id`=`cv`.`id` + and `cv`.`type_id`=1 + )" + )); + + $stats['aussurers_with_test'] = number_format(tc( + "select count(*) as `count` from `users` + where ( + select sum(`points`) from `notary` + where `to`=`users`.`id` + and `deleted` = 0 + ) >= 100 + and exists( + select 1 from `cats_passed` as `cp`, `cats_variant` as `cv` + where `cp`.`user_id`=`users`.`id` + and `cp`.`variant_id`=`cv`.`id` + and `cv`.`type_id`=1 + )" + )); + + $stats['points_issued'] = number_format(tc( + "select sum(greatest(`points`, `awarded`)) as `count` from `notary` + where `deleted` = 0 + and `method` = 'Face to Face Meeting'")); + + $totalusers=0; + $totassurers=0; + $totalcerts=0; + for($i = 0; $i < 12; $i++) { + $first_ts = mktime(0, 0, 0, date("m") - $i, 1, date("Y")); + $next_month_ts = mktime(0, 0, 0, date("m") - $i + 1, 1, date("Y")); + $first = date("Y-m-d", $first_ts); + $next_month = date("Y-m-d", $next_month_ts); + + echo "Calculating statistics for month $first\n"; + + $totalusers += $users = tc( + "select count(*) as `count` from `users` + where `created` >= '$first' and `created` < '$next_month' + and `verified` = 1 + and `deleted` = 0 + and `locked` = 0"); + + $totassurers += $assurers = tc( + "select count(*) as `count` from ( + select 1 from `notary` + where `when` >= '$first' and `when` < '$next_month' + and `method`!='Administrative Increase' + and `deleted` = 0 + group by `to` having sum(`points`) >= 100 + ) as `assurer_candidates`"); + + $certs = tc( + "select count(*) as `count` from `domaincerts` + where `created` >= '$first' and `created` < '$next_month' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `emailcerts` + where `created` >= '$first' and `created` < '$next_month' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `gpg` + where `issued` >= '$first' and `issued` < '$next_month' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `orgdomaincerts` + where `created` >= '$first' and `created` < '$next_month' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `orgemailcerts` + where `created` >= '$first' and `created` < '$next_month' + and `expire` != 0"); + $totalcerts += $certs; + + $tmp_arr = array(); + $tmp_arr['date'] = date("Y-m", $first_ts); + $tmp_arr['new_users'] = number_format($users); + $tmp_arr['new_assurers'] = number_format($assurers); + $tmp_arr['new_certificates'] = number_format($certs); + + $stats['growth_last_12m'][] = $tmp_arr; + } + $stats['growth_last_12m_total'] = array( + 'new_users' => number_format($totalusers), + 'new_assurers' => number_format($totassurers), + 'new_certificates' => number_format($totalcerts), + ); + + $totalcerts = 0; + $totalusers = 0; + $totassurers = 0; + for($i = date("Y"); $i >= 2002; $i--) { + $first_ts = mktime(0, 0, 0, 1, 1, $i); + $next_year_ts = mktime(0, 0, 0, 1, 1, $i + 1); + $first = date("Y-m-d", $first_ts); + $next_year = date("Y-m-d", $next_year_ts); + + echo "Calculating statistics for year $i\n"; + + $totalusers += $users = tc( + "select count(*) as `count` from `users` + where `created` >= '$first' and `created` < '$next_year' + and `verified` = 1 + and `deleted` = 0 + and `locked` = 0"); + + $totassurers += $assurers = tc( + "select count(*) as `count` from ( + select 1 from `notary` + where `when` >= '$first' and `when` < '$next_year' + and `method`!='Administrative Increase' + and `deleted` = 0 + group by `to` having sum(`points`) >= 100 + ) as `assurer_candidates`"); + + $certs = tc( + "select count(*) as `count` from `domaincerts` + where `created` >= '$first' and `created` < '$next_year' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `emailcerts` + where `created` >= '$first' and `created` < '$next_year' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `gpg` + where `issued` >= '$first' and `issued` < '$next_year' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `orgdomaincerts` + where `created` >= '$first' and `created` < '$next_year' + and `expire` != 0"); + $certs += tc( + "select count(*) as `count` from `orgemailcerts` + where `created` >= '$first' and `created` < '$next_year' + and `expire` != 0"); + $totalcerts += $certs; + + $tmp_arr = array(); + $tmp_arr['date'] = $i; + $tmp_arr['new_users'] = number_format($users); + $tmp_arr['new_assurers'] = number_format($assurers); + $tmp_arr['new_certificates'] = number_format($certs); + + $stats['growth_last_years'][] = $tmp_arr; + } + $stats['growth_last_years_total'] = array( + 'new_users' => number_format($totalusers), + 'new_assurers' => number_format($totassurers), + 'new_certificates' => number_format($totalcerts), + ); + + return $stats; +} + + +$stats = getDataFromLive(); +if (! updateCache($stats) ) { + fwrite(STDERR, + "An error occured. The statistics were not successfully updated!"); + die(1); +} diff --git a/www/stats.php b/www/stats.php index 93f693e..1599e17 100644 --- a/www/stats.php +++ b/www/stats.php @@ -1,268 +1,160 @@ - 0) { - $ar = mysql_fetch_assoc($res); - $stats = unserialize($ar['cache']); - $stats['timestamp'] = $ar['timestamp']; - if ($ar['timestamp'] + MAX_CACHE_TTL < time()) - { - $stats=getDataFromLive(); - updateCache($stats); - } - return $stats; - } - $stats=getDataFromLive(); - updateCache($stats); - return $stats; - } - - /** - * get statistics data from live tables, takes a long time so please try to use the - * cache - * @return array - */ - function getDataFromLive() { - $stats = array(); - $stats['verified_users'] = number_format(tc(mysql_query("select count(`id`) as `count` from `users` where `verified`=1"))); - $stats['verified_emails'] = number_format(tc(mysql_query("select count(`id`) as `count` from `email` where `hash`='' and `deleted`=0"))); - $stats['verified_domains'] = number_format(tc(mysql_query("select count(`id`) as `count` from `domains` where `hash`='' and `deleted`=0"))); - $certs = tc(mysql_query("select count(`id`) as `count` from `domaincerts`")); - $certs += tc(mysql_query("select count(`id`) as `count` from `emailcerts`")); - $certs += tc(mysql_query("select count(`id`) as `count` from `gpg`")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgdomaincerts`")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgemailcerts`")); - $stats['verified_certificates'] = number_format($certs); - $certs = tc(mysql_query("select count(`id`) as `count` from `domaincerts` where `revoked`=0 and `expire`>NOW()")); - $certs += tc(mysql_query("select count(`id`) as `count` from `emailcerts` where `revoked`=0 and `expire`>NOW()")); - $certs += tc(mysql_query("select count(`id`) as `count` from `gpg` where `expire`<=NOW()")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgdomaincerts` where `revoked`=0 and `expire`>NOW()")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgemailcerts` where `revoked`=0 and `expire`>NOW()")); - $stats['valid_certificates'] = number_format($certs); - $stats['assurances_made'] = number_format(tc(mysql_query("select count(`id`) as `count` from `notary`"))); - $stats['users_1to49'] = number_format(mysql_num_rows(mysql_query("select `to` from `notary` group by `to` having sum(`points`) > 0 and sum(`points`) < 50"))); - $stats['users_50to99'] = number_format(mysql_num_rows(mysql_query("select `to` from `notary` group by `to` having sum(`points`) >= 50 and sum(`points`) < 100"))); - $stats['assurer_candidates'] = number_format(tc(mysql_query("select count(*) as `count` from `users` where ". - "not exists(select 1 from `cats_passed` as `cp`, `cats_variant` as `cv` where `cp`.`user_id`=`users`.`id` and `cp`.`variant_id`=`cv`.`id` and `cv`.`type_id`=1) and ". - "(select sum(`points`) from `notary` where `to`=`users`.`id`) >= 100"))); - $stats['aussurers_with_test'] = number_format(tc(mysql_query("select count(*) as `count` from `users` where ". - "exists(select 1 from `cats_passed` as `cp`, `cats_variant` as `cv` where `cp`.`user_id`=`users`.`id` and `cp`.`variant_id`=`cv`.`id` and `cv`.`type_id`=1) and ". - "(select sum(`points`) from `notary` where `to`=`users`.`id`) >= 100"))); - $stats['points_issued'] = number_format(tc(mysql_query("select sum(`points`) as `count` from `notary`"))); - - $totalusers=0; - $totassurers=0; - $totalcerts=0; - for($i = 0; $i < 12; $i++) { - $tmp_arr = array(); - $tmp_arr['date'] = date("Y-m", mktime(0,0,0,date("m") - $i,1,date("Y"))); - $date = date("Y-m", mktime(0,0,0,date("m") - $i,1,date("Y"))); - $totalusers += $users = tc(mysql_query("select count(`id`) as `count` from `users` where `created` like '$date%' and `verified`=1")); - $totassurers += $assurers = mysql_num_rows(mysql_query("select `to` from `notary` where `when` like '$date%' and `method`!='Administrative Increase' group by `to` having sum(`points`) >= 100")); - $certs = tc(mysql_query("select count(`id`) as `count` from `domaincerts` where `created` like '$date%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `emailcerts` where `created` like '$date%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `gpg` where `issued` like '$date%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgdomaincerts` where `created` like '$date%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgemailcerts` where `created` like '$date%'")); - $totalcerts += $certs; - - $tmp_arr['new_users'] = number_format($users); - $tmp_arr['new_assurers'] = number_format($assurers); - $tmp_arr['new_certificates'] = number_format($certs); - - $stats['growth_last_12m'][] = $tmp_arr; - } - $stats['growth_last_12m_total'] = array('new_users' => number_format($totalusers), - 'new_assurers' => number_format($totassurers), - 'new_certificates' => number_format($totalcerts)); - - $totalcerts = 0; - $totalusers = 0; - $totassurers = 0; - for($i = date("Y"); $i >= 2002; $i--) { - $tmp_arr = array(); - $tmp_arr['date'] = $i; - $totalusers += $users = tc(mysql_query("select count(`id`) as `count` from `users` where `created` like '$i%' and `verified`=1")); - $totassurers += $assurers = mysql_num_rows(mysql_query("select `to` from `notary` where `when` like '$i%' and `method`!='Administrative Increase' group by `to` having sum(`points`) >= 100")); - $certs = tc(mysql_query("select count(`id`) as `count` from `domaincerts` where `created` like '$i%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `emailcerts` where `created` like '$i%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `gpg` where `issued` like '$i%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgdomaincerts` where `created` like '$i%'")); - $certs += tc(mysql_query("select count(`id`) as `count` from `orgemailcerts` where `created` like '$i%'")); - $totalcerts += $certs; - - $tmp_arr['new_users'] = number_format($users); - $tmp_arr['new_assurers'] = number_format($assurers); - $tmp_arr['new_certificates'] = number_format($certs); - - $stats['growth_last_years'][] = $tmp_arr; - } - $stats['growth_last_years_total'] = array('new_users' => number_format($totalusers), - 'new_assurers' => number_format($totassurers), - 'new_certificates' => number_format($totalcerts)); - - return $stats; - } - - $stats = getData(); -?> -

CAcert.org

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CAcert.org
:
:
:
:
:
:
:
:
:
:
:
-
- - - - - - - - - - - - - - - - - - - - -
CAcert.org
- - - -
N/A
-
- - - - - - - - - - - - - - - - - - - - -
CAcert.org
- - - -
N/A
-
- -
- - - + 0) { + $ar = mysql_fetch_assoc($res); + $stats = unserialize($ar['cache']); + $stats['timestamp'] = $ar['timestamp']; + return $stats; + } + + return null; + } + + $stats = getData(); + if ($stats === null) { + echo '

', _("Error while retrieving the statistics!"), '

'; + showfooter(); + die(); + } +?> +

CAcert.org

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CAcert.org
:
:
:
:
:
:
:
:
:
:
:
+
+ + + + + + + + + + + + + + + + + + + + +
CAcert.org
+ + + +
+
+ + + + + + + + + + + + + + + + + + + + +
CAcert.org
+ + + +
+
+ +
+
+ + +