Add Filtermaster

modules/filtermaster/manifests/init.pp

 class filtermaster {
   Cron {
-    environment => ['MAILTO=ROOT', 'PYTHONPATH=/opt/sitescripts'],

[Wladimir Palant, 2013/11/08 15:32:06]

Add a TODO comment so that you don't forget to change the MAILTO variable before
committing?

[christian, 2013/11/08 16:25:01]

Done.

+   #TODO change email
+    environment => ['MAILTO=root', 'PYTHONPATH=/opt/sitescripts'],
   }
-        
+  
   class {'ssh':
     custom_configuration => 'Match User rsync
     AllowTcpForwarding no
     X11Forwarding no
     AllowAgentForwarding no
     GatewayPorts no
-    ForceCommand rsync --server --sender -vltprz --delete-excluded --exclude CVS . /home/rsync/subscriptions/'

[Wladimir Palant, 2013/11/08 15:32:06]

/home/rsync/subscriptions isn't what you want to sync,
/home/rsync/subscriptions/files/subscriptions is (as per your sitescripts.ini).

--exclude CVS is unnecessary, you don't have CVS syncing any more.

Note: there is a change under review
(http://codereview.adblockplus.org/28037010/), once it is committed there will
be an additional data/ at the end of the path.

[christian, 2013/11/08 16:25:01]

Done.

+    ForceCommand rsync --server --sender -vltprz --delete-excluded . /home/rsync/generated/data/'
   }
   
   user {'rsync':
     ensure => present,
     comment => 'Filter list mirror user',
     home => '/home/rsync',
     managehome => true
   }
-         
+
+  file {'/home/rsync/update_repos.sh':
+    ensure => file,
+    owner => rsync,
+    mode => 0700,
+    source => 'puppet:///modules/filtermaster/update_repos.sh'
+  }
+   
+  file {'/home/rsync/subscription':
+    ensure => directory,
+    owner => rsync
+  }
+
+  file {'/home/rsync/generated':
+    ensure => directory,
+    owner => rsync
+  }
+   
   file {'/home/rsync/.ssh':
     ensure => directory,
-    require => User['rsync'],
     owner => rsync,
     mode => 0600
   }
-         
+
   file {'/home/rsync/.ssh/authorized_keys':
     ensure => file,
-    require => [
-    File['/home/rsync/.ssh'],
-    User['rsync']
-    ],
     owner => rsync,
     mode => 0600,
-    source => 'puppet:///modules/filtermaster/authorized_keys'
+    source => 'puppet:///modules/private/rsync@easylist-downloads.adblockplus.org.pub'
   }
 
-  file {'/etc/sitescripts':
+  file {'/etc/ssh/ssh_host_rsa_key':
+    require => Package['openssh-server'],
+    notify => Service['ssh'],
     ensure => file,
     owner => root,
-    mode => 0644,
-    source => 'puppet:///modules/filtermaster/sitescripts'
+    group => root,
+    source => 'puppet:///modules/private/filtermaster.adblockplus.org_ssh.key'
   }
 
-#donwload the repos

[Wladimir Palant, 2013/11/08 15:32:06]

Typo: donwload => download

[Felix Dahlke, 2013/11/08 15:49:39]

I'd actually remove this, "repo_download" pretty much says it all.

[christian, 2013/11/08 16:25:01]

Done.

+  file {'/etc/ssh/ssh_host_rsa_key.pub':
+    require => Package['openssh-server'],
+    notify => Service['ssh'],
+    ensure => file,
+    owner => root,
+    group => root,
+    source => 'puppet:///modules/private/filtermaster.adblockplus.org_ssh.pub'
+  }
 
-  define repo_download( $name ) { 
-    exec { "fetch_${title}":
-      command =>  "hg clone https://hg.adblockplus.org/${name} /home/rsync/subscription/${name}",
+  package {['p7zip-full']:}
+
+  define repo_download() { 
+    exec {"fetch_${title}":
+      command => "hg clone https://hg.adblockplus.org/${title} /home/rsync/subscription/${title}",
       path => ["/usr/bin/", "/bin/"],
       require => Package['mercurial'],
       user => rsync,
       timeout => 0,
-      onlyif => "test ! -d /home/rsync/subscription/${name}"
+      onlyif => "test ! -d /home/rsync/subscription/${title}"
     }

[Wladimir Palant, 2013/11/08 15:32:06]

What about a cron job to update these repositories?

[christian, 2013/11/08 16:25:01]

Don't "updateSubscriptionDownloads" update all repos? Is there a need for an
additional cron job?

   }
  
-  repo_download {'easylist':
-     name => "easylist"
+  repo_download {['easylist',
+                  'easylistgermany',
+                  'easylistitaly',
+                  'easylistcombinations',
+                  'malwaredomains',
+                  'ruadlist',
+                  'listefr',
+                  'exceptionrules',
+                  'easylistdutch',
+                  'facebookfilters',
+                  'antiadblockfilters'
+     ]:
   }  
 
-  repo_download {'easylist_germany':
-    name => "easylistgermany"

[Wladimir Palant, 2013/11/08 15:32:06]

This looks redundant, why not drop the $name parameter and only use $title? Then
it would simply be:

repo_download {'easylistgermany': }

[Felix Dahlke, 2013/11/08 15:49:39]

Yes, then you could actually do this:

repo_download {['easylistgermany',
                'easylist_italy',
                '...']: }

+  cron {'update_subscription':
+    ensure => present,
+    command => "python -m sitescripts.subscriptions.bin.updateSubscriptionDownloads 3>&1 1>/dev/null 2>&3 | perl -pe 's/^/\"[\" . scalar localtime() . \"] \"/e' >> /tmp/subscription_errors && chmod 666 /tmp/subscription_errors 2>/dev/null",
+    user => rsync,
+    require => User['rsync'],
+    minute => '*/10'
+  }
+     
+  cron {'update_malware':
+    ensure => present,
+    command => "python -m sitescripts.subscriptions.bin.updateMalwareDomainsList",
+    user => rsync,
+    require => User['rsync'],
+    hour => '*/6',
+    minute => 15
+  }
+  
+  cron {'update_repos':
+    ensure => present,
+    command => "/home/rsync/update_repos.sh",
+    user => rsync,
+    require => [
+      User['rsync'],
+      File['/home/rsync/update_repos.sh']
+    ],
+    minute  => '8-58/10'
   }
 
-  repo_download {'easylist_italy':
-    name => "easylistitaly"
+  class {'sitescripts':
+    sitescriptsini_source => 'puppet:///modules/filtermaster/sitescripts'
   }
-
-  repo_download {'easylist_combinations':
-    name => "easylistcombinations"
-  }
-
-  repo_download {'malwaredomains':
-    name => "malwaredomains"
-  }
-
-  repo_download {'ruadlist':
-    name => "ruadlist"
-  }
-
-  repo_download {'listefr':
-    name => "listefr"
-  }
-
-  repo_download {'exceptionrules':
-    name => "exceptionrules"
-  }
-  
-  cron {update_subscription:

[Felix Dahlke, 2013/11/08 15:49:39]

Still needs to go in '' for consistency, same below.

-    ensure => present,
-    command => "python -m sitescripts.subscriptions.bin.updateSubscriptionDownloads 3>&1 1>/dev/null 2>&3 | perl -pe 's/^/\"[\" . scalar localtime() . \"] \"/e' >> /tmp/subscription_errors && chmod 666 /tmp/subscription_errors 2>/dev/null",
-    user    => rsync,
-    require => [
-    User['rsync']
-    ],
-    minute  => '*/10'
-  }
-                 
-  cron {update_malware:
-    ensure => present,
-    command => "python -m sitescripts.subscriptions.bin.updateMalwareDomainsList",
-    user    => rsync,
-    require => [
-    User['rsync']
-    ],
-    hour    => '*/6',
-    minute  => 15
-  }
-}       
+}