Issue 239 - Move AWStats processing to the stats server

modules/statsmaster/manifests/awstats.pp

 class statsmaster::awstats {
   package {['awstats', 'libgeo-ip-perl']:}
 
-  file {'/var/www/awstatsdata':

[Felix Dahlke, 2014/04/25 04:37:08]

How about combining these directory resources?

file {['/var/www/awstatsdata',
       '/var/www/awstatsconf',
       '/var/www/awstats',
       '/var/www/awstats/archive']:
  ensure => directory,
  owner => root,
  mode => 0755
}

-    ensure => directory,
-    owner => root,
-    mode => 0755,
-  }
-
-  file {'/var/www/awstatsconf':
-    ensure => directory,
-    owner => root,
-    mode => 0755,
-  }
-
-  file {'/var/www/awstats':
-    ensure => directory,
-    owner => root,
-    mode => 0755,
-  }
-
-  file {'/var/www/awstats/archive':
+  file {['/var/www/awstatsdata', '/var/www/awstatsconf', '/var/www/awstats']:
     ensure => directory,
     owner => root,
     mode => 0755,
   }
 
   concat {'/var/www/awstats/index.html':
     mode => 0644,
     owner => root,
   }
 
-  concat {'/home/stats/process_logs':
-    owner => stats,
-    mode => 0700,
-  }
-
-  concat {'/home/stats/build_static':
+  concat {['/home/stats/process_logs', '/home/stats/build_static',
+      '/home/stats/anonymize_ips']:
     owner => stats,
     mode => 0700,
   }
 
   concat::fragment {'index_head':
     target => '/var/www/awstats/index.html',
     content => template('statsmaster/index_head.html.erb'),
     order => 'aaahead',
   }
 
   concat::fragment {'index_foot':
     target => '/var/www/awstats/index.html',
     content => template('statsmaster/index_foot.html.erb'),
     order => 'zzzfoot',
   }
 
   concat::fragment {'process_logs_head':
     target => '/home/stats/process_logs',
     content => template('statsmaster/process_logs_head.erb'),
     order => 'aaahead',
   }
 
   concat::fragment {'build_static_head':
     target => '/home/stats/build_static',
     content => template('statsmaster/build_static_head.erb'),
     order => 'aaahead',
   }
 
+  concat::fragment {'anonymize_ips_head':
+    target => '/home/stats/anonymize_ips',
+    content => template('statsmaster/anonymize_ips_head.erb'),
+    order => 'aaahead',
+  }
+
   define siteconfig($host, $log) {
     file {"/var/www/awstatsconf/awstats.$title.conf":
       ensure => present,
       mode => 0444,
       owner => root,
       content => template('statsmaster/awstats.conf'),
     }
 
-    file {["/var/www/awstatsdata/$title", "/var/www/awstats/$title", "/var/www/awstats/archive/$title"]:

[Felix Dahlke, 2014/04/25 04:37:08]

Nit: Wrap at 80 columns?

+    file {["/var/www/awstatsdata/$title", "/var/www/awstats/$title",
+        "/var/www/awstats/archive/$title"]:
       ensure => directory,
       mode => 0755,
       owner => stats,
     }
 
     concat::fragment {"index_$title":
       target => '/var/www/awstats/index.html',
       content => template('statsmaster/index_item.html.erb'),
       order => $title,
     }
 
     concat::fragment {"process_logs_$title":
       target => '/home/stats/process_logs',
       content => template('statsmaster/process_logs_item.erb'),
       order => $title,
     }
 
     concat::fragment {"build_static_$title":
       target => '/home/stats/build_static',
       content => template('statsmaster/build_static_item.erb'),
       order => $title,
     }
+
+    concat::fragment {"anonymize_ips_$title":
+      target => '/home/stats/anonymize_ips',
+      content => template('statsmaster/anonymize_ips_item.erb'),
+      order => $title,
+    }
   }
 
   $sites = {
     'adblockplus.org' => {
       host => 'ssh.adblockplus.org',
       log => 'access_log_adblockplus',
     },
     'easylist.adblockplus.org' => {
       host => 'ssh.adblockplus.org',
       log => 'access_log_easylist',
     },
     'share.adblockplus.org' => {
       host => 'ssh.adblockplus.org',
       log => 'access_log_share',
     },
     'facebook.adblockplus.me' => {
       host => 'ssh.adblockplus.org',
       log => 'access_log_facebook',
     },
-    'youtube.adblockplus.org' => {

[Wladimir Palant, 2014/04/24 18:57:03]

The site is called youtube.adblockplus.me, not youtube.adblockplus.org, fixed
that in https://hg.adblockplus.org/infrastructure/rev/ea9e282d51b1

+    'youtube.adblockplus.me' => {
       host => 'ssh.adblockplus.org',
       log => 'access_log_youtube',
     },
     'acceptableads.org' => {
       host => 'ssh.adblockplus.org',
       log => 'access_log_acceptableads',
     },
     'eyeo.com' => {
       host => 'web1.adblockplus.org',
       log => 'access_log_eyeo.com',
     },
     'intraforum.adblockplus.org' => {
-      host => 'server10.adblockplus.org',

[Wladimir Palant, 2014/04/24 18:57:03]

The DNS name is server_10, fixed that in
https://hg.adblockplus.org/infrastructure/rev/eee27e75236e

+      host => 'server_10.adblockplus.org',
       log => 'access_log_intraforum',
     },
   }
 
   create_resources(statsmaster::awstats::siteconfig, $sites)
 
   cron {'awstats_update':
     ensure => present,
     require => [
       Package['awstats', 'libgeo-ip-perl'],
-      Concat['/home/stats/process_logs'],
-      Concat['/home/stats/build_static'],
-      File['/var/www/awstatsconf'],
-      File['/var/www/awstatsdata'],
-      File['/var/www/awstats'],
+      File['/home/stats/process_logs', '/home/stats/build_static',
+        '/var/www/awstatsconf', '/var/www/awstatsdata', '/var/www/awstats'],
     ],
     command => '/home/stats/process_logs && /home/stats/build_static',
     environment => ['MAILTO=admins@adblockplus.org,root'],
     user => stats,
     hour => 4,
     minute => 0,
   }
 
   cron {'awstats_prevmonth':
     ensure => present,
     require => [
       Package['awstats'],
-      Concat['/home/stats/build_static'],
-      File['/var/www/awstatsconf'],
-      File['/var/www/awstatsdata'],
-      File['/var/www/awstats/archive'],
+      File['/home/stats/build_static', '/home/stats/anonymize_ips',
+        '/var/www/awstatsconf', '/var/www/awstatsdata', '/var/www/awstats'],
     ],
-    command => '/home/stats/build_static prevmonth',
+    command => '/home/stats/anonymize_ips prevmonth && /home/stats/build_static prevmonth',
     environment => ['MAILTO=admins@adblockplus.org,root'],
     user => stats,
     monthday => 1,
     hour => 6,
     minute => 0,
   }
 }