From ee57b5921f59e7f49dabdb37d8ae93d8a7744aab Mon Sep 17 00:00:00 2001
From: andryyy <andre.peters@debinux.de>
Date: Sat, 12 Oct 2019 13:14:34 +0200
Subject: [PATCH] [Rspamd] Various fixes for Rspamd 2.0, neural network
 activated, autolearning activated (auto-keeps a ratio)

---
 data/conf/rspamd/local.d/neural.conf       | 39 +++++++++++++++-------
 data/conf/rspamd/local.d/neural_group.conf | 18 +++++++---
 data/conf/rspamd/local.d/statistic.conf    | 15 ++++++---
 data/conf/rspamd/lua/rspamd.local.lua      |  3 +-
 data/conf/rspamd/override.d/logging.inc    |  1 -
 5 files changed, 52 insertions(+), 24 deletions(-)

diff --git a/data/conf/rspamd/local.d/neural.conf b/data/conf/rspamd/local.d/neural.conf
index eb98488d..d1ec510b 100644
--- a/data/conf/rspamd/local.d/neural.conf
+++ b/data/conf/rspamd/local.d/neural.conf
@@ -1,13 +1,28 @@
-train {
-  max_train = 1k; # Number of trains per epoch
-  max_usages = 50; # Number of learn iterations while ANN data is valid
-  spam_score = 12; # Score to learn spam
-  ham_score = -7; # Score to learn ham
-  learning_rate = 0.01; # Rate of learning (Torch only)
-  max_iterations = 25; # Maximum iterations of learning (Torch only)
+rules {
+  "LONG" {
+    train {
+      max_trains = 200;
+      max_usages = 20;
+      max_iterations = 25;
+      learning_rate = 0.01,
+      spam_score = 12;
+      ham_score = -7;
+    }
+    symbol_spam = "NEURAL_SPAM_LONG";
+    symbol_ham = "NEURAL_HAM_LONG";
+    ann_expire = 45d;
+  }
+  "SHORT" {
+    train {
+      max_trains = 100;
+      max_usages = 10;
+      max_iterations = 15;
+      learning_rate = 0.01,
+      spam_score = 9;
+      ham_score = -3;
+    }
+    symbol_spam = "NEURAL_SPAM_SHORT";
+    symbol_ham = "NEURAL_HAM_SHORT";
+    ann_expire = 7d;
+  }
 }
-ann_expire = 80d;
-timeout = 20; # Increase redis timeout
-enabled = false;
-use_settings = false; # If enabled, then settings-id is used to dispatch networks
-
diff --git a/data/conf/rspamd/local.d/neural_group.conf b/data/conf/rspamd/local.d/neural_group.conf
index 8d43d715..fca5cec9 100644
--- a/data/conf/rspamd/local.d/neural_group.conf
+++ b/data/conf/rspamd/local.d/neural_group.conf
@@ -1,10 +1,18 @@
 symbols = {
-  "NEURAL_SPAM" {
-    weight = 4.0; # sample weight
-    description = "Neural network spam";
+  "NEURAL_SPAM_LONG" {
+    weight = 3.7; # sample weight
+    description = "Neural network spam (long)";
   }
-  "NEURAL_HAM" {
+  "NEURAL_HAM_LONG" {
     weight = -4.0; # sample weight
-    description = "Neural network ham";
+    description = "Neural network ham (long)";
+  }
+  "NEURAL_SPAM_SHORT" {
+    weight = 2.5; # sample weight
+    description = "Neural network spam (short)";
+  }
+  "NEURAL_HAM_SHORT" {
+    weight = -2.0; # sample weight
+    description = "Neural network ham (short)";
   }
 }
diff --git a/data/conf/rspamd/local.d/statistic.conf b/data/conf/rspamd/local.d/statistic.conf
index 4867f78a..1ca3e082 100644
--- a/data/conf/rspamd/local.d/statistic.conf
+++ b/data/conf/rspamd/local.d/statistic.conf
@@ -2,18 +2,23 @@ classifier "bayes" {
     tokenizer {
       name = "osb";
     }
-    autolearn = false;
     backend = "redis";
     min_tokens = 11;
     min_learns = 5;
     new_schema = true;
     expire = 2592000;
     statfile {
-        symbol = "BAYES_HAM";
-        spam = false;
+      symbol = "BAYES_HAM";
+      spam = false;
     }
     statfile {
-        symbol = "BAYES_SPAM";
-        spam = true;
+      symbol = "BAYES_SPAM";
+      spam = true;
+    }
+    autolearn {
+      spam_threshold = 12.0;
+      ham_threshold = -4.5;
+      check_balance = true;
+      min_balance = 0.9;
     }
 }
diff --git a/data/conf/rspamd/lua/rspamd.local.lua b/data/conf/rspamd/lua/rspamd.local.lua
index c6fc0b98..41db2590 100644
--- a/data/conf/rspamd/lua/rspamd.local.lua
+++ b/data/conf/rspamd/lua/rspamd.local.lua
@@ -23,7 +23,7 @@ rspamd_config:register_symbol({
     local redis_params = rspamd_parse_redis_server('keep_spam')
     local ip = task:get_from_ip()
 
-    if not ip:is_valid() then
+    if ip == nil or not ip:is_valid() then
       return false
     end
 
@@ -174,6 +174,7 @@ rspamd_config:register_symbol({
     end
     return true
   end,
+  flags = 'empty',
   priority = 20
 })
 
diff --git a/data/conf/rspamd/override.d/logging.inc b/data/conf/rspamd/override.d/logging.inc
index 7765bb34..23a9f3cf 100644
--- a/data/conf/rspamd/override.d/logging.inc
+++ b/data/conf/rspamd/override.d/logging.inc
@@ -1,5 +1,4 @@
 type = "console";
 systemd = false;
-level = "silent";
 .include "$CONFDIR/logging.inc"
 .include(try=true; priority=20) "$CONFDIR/override.d/logging.custom.inc"