From 873222d5f8f9efc3355b1babe394a543698bcfa4 Mon Sep 17 00:00:00 2001
From: "andre.peters" <andre.peters@servercow.de>
Date: Sat, 9 Dec 2017 09:08:18 +0100
Subject: [PATCH] [Rspamd] Remove DKIM forced action, move ratelimit lua, add
 meta exporter

---
 data/conf/rspamd/custom/.empty                |   1 -
 data/conf/rspamd/dynmaps/settings.php         |   2 +-
 data/conf/rspamd/local.d/composites.conf      |   8 +-
 data/conf/rspamd/local.d/force_actions.conf   |  15 --
 data/conf/rspamd/local.d/statistic.conf       |   3 +-
 .../conf/rspamd/{custom => lua}/ratelimit.lua |   0
 data/conf/rspamd/meta_exporter/pipe.php       | 155 ++++++++++++++++++
 data/conf/rspamd/meta_exporter/vars.inc.php   |   3 +
 data/conf/rspamd/override.d/logging.inc       |   1 +
 data/conf/rspamd/override.d/ratelimit.conf    |   2 +-
 .../rspamd/override.d/worker-controller.inc   |   3 +-
 data/conf/rspamd/override.d/worker-normal.inc |   1 +
 data/conf/rspamd/override.d/worker-proxy.inc  |   1 +
 13 files changed, 172 insertions(+), 23 deletions(-)
 delete mode 100644 data/conf/rspamd/custom/.empty
 rename data/conf/rspamd/{custom => lua}/ratelimit.lua (100%)
 create mode 100644 data/conf/rspamd/meta_exporter/pipe.php
 create mode 100644 data/conf/rspamd/meta_exporter/vars.inc.php

diff --git a/data/conf/rspamd/custom/.empty b/data/conf/rspamd/custom/.empty
deleted file mode 100644
index d00491fd..00000000
--- a/data/conf/rspamd/custom/.empty
+++ /dev/null
@@ -1 +0,0 @@
-1
diff --git a/data/conf/rspamd/dynmaps/settings.php b/data/conf/rspamd/dynmaps/settings.php
index 335c0c66..fcc656a8 100644
--- a/data/conf/rspamd/dynmaps/settings.php
+++ b/data/conf/rspamd/dynmaps/settings.php
@@ -17,7 +17,7 @@ $opt = [
 ];
 try {
   $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-  $stmt = $pdo->query("SELECT * FROM `filterconf`");
+  $stmt = $pdo->query("SELECT '1' FROM `filterconf`");
 }
 catch (PDOException $e) {
   echo 'settings { }';
diff --git a/data/conf/rspamd/local.d/composites.conf b/data/conf/rspamd/local.d/composites.conf
index e895fb0f..ea35dc92 100644
--- a/data/conf/rspamd/local.d/composites.conf
+++ b/data/conf/rspamd/local.d/composites.conf
@@ -1,4 +1,8 @@
 MX_IMPLICIT {
-    expression = "MX_GOOD and MX_MISSING";
-    score = -0.01;
+  expression = "MX_GOOD and MX_MISSING";
+  score = -0.01;
+}
+VIRUS_FOUND {
+  expression = "CLAM_VIRUS & !MAILCOW_WHITE";
+  score = 2000;
 }
diff --git a/data/conf/rspamd/local.d/force_actions.conf b/data/conf/rspamd/local.d/force_actions.conf
index 956402f5..a1b9899d 100644
--- a/data/conf/rspamd/local.d/force_actions.conf
+++ b/data/conf/rspamd/local.d/force_actions.conf
@@ -1,14 +1,4 @@
 rules {
-  DKIM_FAIL {
-    action = "add header";
-    expression = "R_DKIM_REJECT & !MAILLIST & !MAILCOW_WHITE & !MAILCOW_BLACK";
-    require_action = ["no action", "greylist", "soft reject"];
-  }
-  VIRUS_FOUND {
-    action = "reject";
-    expression = "CLAM_VIRUS & !MAILCOW_WHITE";
-    honor_action = ["reject"];
-  }
   WHITELIST_FORWARDING_HOST_NO_REJECT {
     action = "add header";
     expression = "WHITELISTED_FWD_HOST";
@@ -19,9 +9,4 @@ rules {
     expression = "WHITELISTED_FWD_HOST";
     require_action = ["greylist", "soft reject"];
   }
-  ADD_UNAUTH_SUBJ {
-    action = "rewrite subject";
-    subject = "[Unauth] %s";
-    expression = "SPOOFED_SENDER";
-  }
 }
diff --git a/data/conf/rspamd/local.d/statistic.conf b/data/conf/rspamd/local.d/statistic.conf
index 4b410842..b66f5018 100644
--- a/data/conf/rspamd/local.d/statistic.conf
+++ b/data/conf/rspamd/local.d/statistic.conf
@@ -7,8 +7,7 @@ classifier "bayes" {
     servers = "redis:6379";
     min_tokens = 11;
     min_learns = 20;
-    autolearn = true;
-
+    autolearn = [-20, 50];
     per_user = <<EOD
 return function(task)
     local rcpt = task:get_recipients(1)
diff --git a/data/conf/rspamd/custom/ratelimit.lua b/data/conf/rspamd/lua/ratelimit.lua
similarity index 100%
rename from data/conf/rspamd/custom/ratelimit.lua
rename to data/conf/rspamd/lua/ratelimit.lua
diff --git a/data/conf/rspamd/meta_exporter/pipe.php b/data/conf/rspamd/meta_exporter/pipe.php
new file mode 100644
index 00000000..1075032c
--- /dev/null
+++ b/data/conf/rspamd/meta_exporter/pipe.php
@@ -0,0 +1,155 @@
+<?php
+// File size is limited by Nginx site to 10M
+// To speed things up, we do not include prerequisites
+header('Content-Type: text/plain');
+require_once "vars.inc.php";
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+// Init database
+$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  http_response_code(501);
+  exit;
+}
+// Init Redis
+$redis = new Redis();
+$redis->connect('redis-mailcow', 6379);
+
+// Functions
+function parse_email($email) {
+  if(!filter_var($email, FILTER_VALIDATE_EMAIL)) return false;
+  $a = strrpos($email, '@');
+  return array('local' => substr($email, 0, $a), 'domain' => substr(substr($email, $a), 1));
+}
+if (!function_exists('getallheaders'))  {
+  function getallheaders() {
+    if (!is_array($_SERVER)) {
+      return array();
+    }
+    $headers = array();
+    foreach ($_SERVER as $name => $value) {
+      if (substr($name, 0, 5) == 'HTTP_') {
+        $headers[str_replace(' ', '-', ucwords(strtolower(str_replace('_', ' ', substr($name, 5)))))] = $value;
+      }
+    }
+    return $headers;
+  }
+}
+
+$raw_data = file_get_contents('php://input');
+$headers = getallheaders();
+
+$qid      = $headers['X-Rspamd-Qid'];
+$score    = $headers['X-Rspamd-Score'];
+$rcpts    = $headers['X-Rspamd-Rcpt'];
+$user     = $headers['X-Rspamd-User'];
+$ip       = $headers['X-Rspamd-Ip'];
+$action   = $headers['X-Rspamd-Action'];
+$sender   = $headers['X-Rspamd-From'];
+$symbols  = $headers['X-Rspamd-Symbols'];
+
+$raw_size = (int)$_SERVER['CONTENT_LENGTH'];
+
+try {
+  if ($max_size = $redis->Get('Q_MAX_SIZE')) {
+    if (!empty($max_size) && ($max_size * 1048576) < $raw_size) {
+      error_log(sprintf("Message too large: %d exceeds %d", $raw_size, ($max_size * 1048576)));
+      http_response_code(505);
+      exit;
+    }
+  }
+  if ($exclude_domains = $redis->Get('Q_EXCLUDE_DOMAINS')) {
+    $exclude_domains = json_decode($exclude_domains, true);
+  }
+  $retention_size = (int)$redis->Get('Q_RETENTION_SIZE');
+}
+catch (RedisException $e) {
+  error_log($e);
+  http_response_code(504);
+  exit;
+}
+
+$filtered_rcpts = array();
+foreach (json_decode($rcpts, true) as $rcpt) {
+  $parsed_mail = parse_email($rcpt);
+  if (in_array($parsed_mail['domain'], $exclude_domains)) {
+    error_log(sprintf("Skipped domain %s", $parsed_mail['domain']));
+    continue;
+  }
+  try {
+    $stmt = $pdo->prepare("SELECT `goto` FROM `alias`
+      WHERE
+      (
+        `address` = :rcpt
+        OR
+        `address` IN (
+          SELECT username FROM mailbox, alias_domain
+            WHERE (alias_domain.alias_domain = :domain_part
+              AND mailbox.username = CONCAT(:local_part, '@', alias_domain.target_domain)
+              AND mailbox.active = '1'
+              AND alias_domain.active='1')
+        )
+      )
+      AND `active`= '1';");
+    $stmt->execute(array(
+      ':rcpt' => $rcpt,
+      ':local_part' => $parsed_mail['local'],
+      ':domain_part' => $parsed_mail['domain']
+    ));
+    $gotos = $stmt->fetch(PDO::FETCH_ASSOC)['goto'];
+    if (!empty($gotos)) {
+      $filtered_rcpts  = array_unique(array_merge($filtered_rcpts, explode(',', $gotos)));
+    }
+  }
+  catch (PDOException $e) {
+    error_log($e->getMessage());
+    http_response_code(502);
+    exit;
+  }
+}
+foreach ($filtered_rcpts as $rcpt) {
+  
+  try {
+    $stmt = $pdo->prepare("INSERT INTO `quarantaine` (`qid`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`)
+      VALUES (:qid, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action)");
+    $stmt->execute(array(
+      ':qid' => $qid,
+      ':score' => $score,
+      ':sender' => $sender,
+      ':rcpt' => $rcpt,
+      ':symbols' => $symbols,
+      ':user' => $user,
+      ':ip' => $ip,
+      ':msg' => $raw_data,
+      ':action' => $action
+    ));
+    $stmt = $pdo->prepare('DELETE FROM `quarantaine` WHERE `id` NOT IN ( 
+      SELECT `id`
+      FROM (
+        SELECT `id`
+        FROM `quarantaine`
+        WHERE `rcpt` = :rcpt
+        ORDER BY id DESC
+        LIMIT :retention_size
+      ) x 
+    );');
+    $stmt->execute(array(
+      ':rcpt' => $rcpt,
+      ':retention_size' => $retention_size
+    ));
+  }
+  catch (PDOException $e) {
+    error_log($e->getMessage());
+    http_response_code(503);
+    exit;
+  }
+}
+
diff --git a/data/conf/rspamd/meta_exporter/vars.inc.php b/data/conf/rspamd/meta_exporter/vars.inc.php
new file mode 100644
index 00000000..d47e9079
--- /dev/null
+++ b/data/conf/rspamd/meta_exporter/vars.inc.php
@@ -0,0 +1,3 @@
+<?php
+require_once('../../../web/inc/vars.inc.php');
+?>
diff --git a/data/conf/rspamd/override.d/logging.inc b/data/conf/rspamd/override.d/logging.inc
index 64a2b7d4..23a9f3cf 100644
--- a/data/conf/rspamd/override.d/logging.inc
+++ b/data/conf/rspamd/override.d/logging.inc
@@ -1,3 +1,4 @@
 type = "console";
 systemd = false;
 .include "$CONFDIR/logging.inc"
+.include(try=true; priority=20) "$CONFDIR/override.d/logging.custom.inc"
diff --git a/data/conf/rspamd/override.d/ratelimit.conf b/data/conf/rspamd/override.d/ratelimit.conf
index 3c0a55c3..f9e359bc 100644
--- a/data/conf/rspamd/override.d/ratelimit.conf
+++ b/data/conf/rspamd/override.d/ratelimit.conf
@@ -9,6 +9,6 @@ rates {
 }
 whitelisted_rcpts = "postmaster,mailer-daemon";
 max_rcpt = 5;
-custom_keywords = "/etc/rspamd/custom/ratelimit.lua";
+custom_keywords = "/etc/rspamd/lua/ratelimit.lua";
 user_keywords = ["user", "customrl"];
 dynamic_rates = { customrl = "customrl"}
diff --git a/data/conf/rspamd/override.d/worker-controller.inc b/data/conf/rspamd/override.d/worker-controller.inc
index ae136461..22d9a024 100644
--- a/data/conf/rspamd/override.d/worker-controller.inc
+++ b/data/conf/rspamd/override.d/worker-controller.inc
@@ -1,8 +1,9 @@
 bind_socket = "*:11334";
-enable_password = "$2$pppq86q9uns51zd5ekfxecj7bxwaefo3$p7f9xdhamydjhtypcr639it3kqeiknx3dk9on7skjypyi8uwwcmy";
 secure_ip = "192.168.0.0/16";
 secure_ip = "172.16.0.0/12";
 secure_ip = "10.0.0.0/8";
 secure_ip = "127.0.0.1";
 secure_ip = "::1";
 secure_ip = "fd4d:6169:6c63:6f77::/64"
+.include(try=true; priority=10) "$CONFDIR/override.d/worker-controller-password.inc"
+.include(try=true; priority=20) "$CONFDIR/override.d/worker-controller.custom.inc" 
diff --git a/data/conf/rspamd/override.d/worker-normal.inc b/data/conf/rspamd/override.d/worker-normal.inc
index 71569636..a7ab4baf 100644
--- a/data/conf/rspamd/override.d/worker-normal.inc
+++ b/data/conf/rspamd/override.d/worker-normal.inc
@@ -1,2 +1,3 @@
 bind_socket = "*:11333";
 task_timeout = 12s;
+.include(try=true; priority=20) "$CONFDIR/override.d/worker-normal.custom.inc"
diff --git a/data/conf/rspamd/override.d/worker-proxy.inc b/data/conf/rspamd/override.d/worker-proxy.inc
index b87c7f29..0df926a7 100644
--- a/data/conf/rspamd/override.d/worker-proxy.inc
+++ b/data/conf/rspamd/override.d/worker-proxy.inc
@@ -5,3 +5,4 @@ upstream {
   default = true;
   hosts = "rspamd:11333"
 }
+.include(try=true; priority=20) "$CONFDIR/override.d/worker-proxy.custom.inc"