[Web] Show ratelimited messages, allow to delete Redis hash to reset status of a bucket

data/web/debug.php

@@ -22,6 +22,7 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
         <li role="presentation"><a href="#tab-watchdog-logs" aria-controls="tab-watchdog-logs" role="tab" data-toggle="tab">Watchdog</a></li>
         <li role="presentation"><a href="#tab-acme-logs" aria-controls="tab-acme-logs" role="tab" data-toggle="tab">ACME</a></li>
         <li role="presentation"><a href="#tab-api-logs" aria-controls="tab-api-logs" role="tab" data-toggle="tab">API</a></li>
+        <li role="presentation"><a href="#tab-api-rl" aria-controls="tab-api-rl" role="tab" data-toggle="tab">Ratelimits</a></li>
         <li role="presentation"><span class="dropdown-desc"><?=$lang['debug']['external_logs'];?></span></li>
         <li role="presentation"><a href="#tab-rspamd-history" aria-controls="tab-rspamd-history" role="tab" data-toggle="tab">Rspamd</a></li>
         <li role="presentation"><span class="dropdown-desc"><?=$lang['debug']['static_logs'];?></span></li>
@@ -272,6 +273,24 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
           </div>
         </div>
 
+        <div role="tabpanel" class="tab-pane" id="tab-api-rl">
+          <div class="panel panel-default">
+            <div class="panel-heading">Ratelimits <span class="badge badge-info table-lines"></span>
+              <div class="btn-group pull-right">
+                <button class="btn btn-xs btn-default add_log_lines" data-post-process="rllog" data-table="rl_log" data-log-url="ratelimited" data-nrows="100">+ 100</button>
+                <button class="btn btn-xs btn-default add_log_lines" data-post-process="rllog" data-table="rl_log" data-log-url="ratelimited" data-nrows="1000">+ 1000</button>
+                <button class="btn btn-xs btn-default refresh_table" data-draw="draw_rl_logs" data-table="rl_log"><?=$lang['admin']['refresh'];?></button>
+              </div>
+            </div>
+            <div class="panel-body">
+              <p class="help-block"><?=$lang['admin']['hash_remove_info'];?></p>
+              <div class="table-responsive">
+                <table class="table table-striped table-condensed" id="rl_log"></table>
+              </div>
+            </div>
+          </div>
+        </div>
+
       </div> <!-- /tab-content -->
     </div> <!-- /col-md-12 -->
   </div> <!-- /row -->

data/web/inc/functions.inc.php

@@ -1263,7 +1263,7 @@ function get_u2f_registrations($username) {
   $sel->execute(array($username));
   return $sel->fetchAll(PDO::FETCH_OBJ);
 }
-function get_logs($container, $lines = false) {
+function get_logs($application, $lines = false) {
   if ($lines === false) {
     $lines = $GLOBALS['LOG_LINES'] - 1; 
   }
@@ -1283,7 +1283,7 @@ function get_logs($container, $lines = false) {
 		return false;
 	}
   // SQL
-  if ($container == "mailcow-ui") {
+  if ($application == "mailcow-ui") {
     if (isset($from) && isset($to)) {
       $stmt = $pdo->prepare("SELECT * FROM `logs` ORDER BY `id` DESC LIMIT :from, :to");
       $stmt->execute(array(
@@ -1304,7 +1304,7 @@ function get_logs($container, $lines = false) {
     }
   }
   // Redis
-  if ($container == "dovecot-mailcow") {
+  if ($application == "dovecot-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('DOVECOT_MAILLOG', $from - 1, $to - 1);
     }
@@ -1318,7 +1318,7 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "postfix-mailcow") {
+  if ($application == "postfix-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('POSTFIX_MAILLOG', $from - 1, $to - 1);
     }
@@ -1332,7 +1332,7 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "sogo-mailcow") {
+  if ($application == "sogo-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('SOGO_LOG', $from - 1, $to - 1);
     }
@@ -1346,7 +1346,7 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "watchdog-mailcow") {
+  if ($application == "watchdog-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('WATCHDOG_LOG', $from - 1, $to - 1);
     }
@@ -1360,7 +1360,7 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "acme-mailcow") {
+  if ($application == "acme-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('ACME_LOG', $from - 1, $to - 1);
     }
@@ -1374,7 +1374,21 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "api-mailcow") {
+  if ($application == "ratelimited") {
+    if (isset($from) && isset($to)) {
+      $data = $redis->lRange('RL_LOG', $from - 1, $to - 1);
+    }
+    else {
+      $data = $redis->lRange('RL_LOG', 0, $lines);
+    }
+    if ($data) {
+      foreach ($data as $json_line) {
+        $data_array[] = json_decode($json_line, true);
+      }
+      return $data_array;
+    }
+  }
+  if ($application == "api-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('API_LOG', $from - 1, $to - 1);
     }
@@ -1388,7 +1402,7 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "netfilter-mailcow") {
+  if ($application == "netfilter-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('NETFILTER_LOG', $from - 1, $to - 1);
     }
@@ -1402,7 +1416,7 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "autodiscover-mailcow") {
+  if ($application == "autodiscover-mailcow") {
     if (isset($from) && isset($to)) {
       $data = $redis->lRange('AUTODISCOVER_LOG', $from - 1, $to - 1);
     }
@@ -1416,7 +1430,7 @@ function get_logs($container, $lines = false) {
       return $data_array;
     }
   }
-  if ($container == "rspamd-history") {
+  if ($application == "rspamd-history") {
     $curl = curl_init();
     curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, '/var/lib/rspamd/rspamd.sock');
     if (!is_numeric($lines)) {

data/web/inc/functions.ratelimit.inc.php

@@ -192,5 +192,44 @@ function ratelimit($_action, $_scope, $_data = null) {
         break;
       }
     break;
+    case 'delete':
+      $data['hash'] = $_data;
+      if ($_SESSION['mailcow_cc_role'] != 'admin' || !preg_match('/^RL[0-9A-Za-z=]+$/i', trim($data['hash']))) {
+        $_SESSION['return'][] = array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+          'msg' => 'access_denied'
+        );
+        return false;
+      }
+      try {
+        if ($redis->exists($data['hash'])) {
+          $redis->delete($data['hash']);
+          $_SESSION['return'][] = array(
+            'type' => 'success',
+            'log' => array(__FUNCTION__, $_action, $_scope, $_data_log),
+            'msg' => 'hash_deleted'
+          );
+          return true;
+        }
+        else {
+          $_SESSION['return'][] = array(
+            'type' => 'warning',
+            'log' => array(__FUNCTION__, $_action, $_scope, $_data_log),
+            'msg' => 'hash_not_found'
+          );
+          return false;
+        }
+      }
+      catch (RedisException $e) {
+        $_SESSION['return'][] = array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $_action, $_scope, $_data_log),
+          'msg' => array('redis_error', $e)
+        );
+        return false;
+      }
+      return false;
+    break;
   }
 }

data/web/js/admin.js

@@ -8,7 +8,7 @@ jQuery(function($){
   $("#rspamd_preset_1").on('click', function(e) {
     e.preventDefault();
     $("form[data-id=rsetting]").find("#adminRspamdSettingsDesc").val(lang.rsettings_preset_1);
-    $("form[data-id=rsetting]").find("#adminRspamdSettingsContent").val('priority = 10;\nauthenticated = yes;\napply "default" {\n  symbols_enabled = ["DKIM_SIGNED", "RATELIMIT_UPDATE", "RATELIMIT_CHECK", "DYN_RL_CHECK", "HISTORY_SAVE", "MILTER_HEADERS", "ARC_SIGNED"];\n}');
+    $("form[data-id=rsetting]").find("#adminRspamdSettingsContent").val('priority = 10;\nauthenticated = yes;\napply "default" {\n  symbols_enabled = ["DKIM_SIGNED", "RATELIMITED", "RATELIMIT_UPDATE", "RATELIMIT_CHECK", "DYN_RL_CHECK", "HISTORY_SAVE", "MILTER_HEADERS", "ARC_SIGNED"];\n}');
   });
   $("#rspamd_preset_2").on('click', function(e) {
     e.preventDefault();

data/web/js/debug.js

@@ -8,6 +8,8 @@ jQuery(function($){
   var entityMap={"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;","/":"&#x2F;","`":"&#x60;","=":"&#x3D;"};
   function escapeHtml(n){return String(n).replace(/[&<>"'`=\/]/g,function(n){return entityMap[n]})}
   function humanFileSize(i){if(Math.abs(i)<1024)return i+" B";var B=["KiB","MiB","GiB","TiB","PiB","EiB","ZiB","YiB"],e=-1;do{i/=1024,++e}while(Math.abs(i)>=1024&&e<B.length-1);return i.toFixed(1)+" "+B[e]}
+  function hashCode(t){for(var n=0,r=0;r<t.length;r++)n=t.charCodeAt(r)+((n<<5)-n);return n}
+  function intToRGB(t){var n=(16777215&t).toString(16).toUpperCase();return"00000".substring(0,6-n.length)+n}
   $(".refresh_table").on('click', function(e) {
     e.preventDefault();
     var table_name = $(this).data('table');
@@ -162,6 +164,48 @@ jQuery(function($){
       }
     });
   }
+  function draw_rl_logs() {
+    ft_rl_logs = FooTable.init('#rl_log', {
+      "columns": [
+        {"name":"indicator","title":" ","style":{"width":"50px"}},
+        {"name":"time","formatter":function unix_time_format(tm) { var date = new Date(tm ? tm * 1000 : 0); return date.toLocaleString();},"title":lang.last_applied,"style":{"width":"170px"}},
+        {"name":"rl_name","title":lang.rate_name},
+        {"name":"from","title":lang.sender},
+        {"name":"rcpt","title":lang.recipients},
+        {"name":"user","title":lang.authed_user},
+        {"name":"message_id","title":"Msg ID","breakpoints": "all","style":{"word-break":"break-all"}},
+        {"name":"header_from","title":"Header From","breakpoints": "all","style":{"word-break":"break-all"}},
+        {"name":"header_subject","title":"Subject","breakpoints": "all","style":{"word-break":"break-all"}},
+        {"name":"rl_hash","title":"Hash","breakpoints": "all","style":{"word-break":"break-all"}},
+        {"name":"qid","title":"Rspamd QID","breakpoints": "all","style":{"word-break":"break-all"}},
+        {"name":"ip","title":"IP","breakpoints": "all","style":{"word-break":"break-all"}},
+        {"name":"action","title":lang.action,"breakpoints": "all","style":{"word-break":"break-all"}},
+      ],
+      "rows": $.ajax({
+        dataType: 'json',
+        url: '/api/v1/get/logs/ratelimited',
+        jsonp: false,
+        error: function () {
+          console.log('Cannot draw rl log table');
+        },
+        success: function (data) {
+          return process_table_data(data, 'rllog');
+        }
+      }),
+      "empty": lang.empty,
+      "paging": {"enabled": true,"limit": 5,"size": log_pagination_size},
+      "filtering": {"enabled": true,"delay": 1,"position": "left","connectors": false,"placeholder": lang.filter_table},
+      "sorting": {"enabled": true},
+      "on": {
+        "ready.ft.table": function(e, ft){
+          table_log_ready(ft, 'rl_logs');
+        },
+        "after.ft.paging": function(e, ft){
+          table_log_paging(ft, 'rl_logs');
+        }
+      }
+    });
+  }
   function draw_ui_logs() {
     ft_api_logs = FooTable.init('#ui_logs', {
       "columns": [
@@ -540,6 +584,19 @@ jQuery(function($){
           item.method = '<span class="label label-warning">' + item.method + '</span>';
         }
       });
+    } else if (table == 'rllog') {
+      $.each(data, function (i, item) {
+        if (item.user == null) {
+          item.user = "none";
+        }
+        if (item.rl_hash == null) {
+          item.rl_hash = "err";
+        }
+        item.indicator = '<span class="label" style="background-color:#' + intToRGB(hashCode(item.rl_hash)) + '!important;padding:3px 5px 0px 5px;">&nbsp;&nbsp;</span>';
+        if (item.rl_hash != 'err') {
+          item.action = '<a href="#" data-action="delete_selected" data-id="single-hash" data-api-url="delete/rlhash" data-item="' + encodeURI(item.rl_hash) + '" class="btn btn-xs btn-danger"><span class="glyphicon glyphicon-trash"></span> ' + lang.reset_limit + '</a>';
+        }
+      });
     }
     return data
   };
@@ -575,6 +632,7 @@ jQuery(function($){
   draw_watchdog_logs();
   draw_acme_logs();
   draw_api_logs();
+  draw_rl_logs();
   draw_ui_logs();
   draw_netfilter_logs();
   draw_rspamd_history();

data/web/json_api.php

@@ -387,6 +387,17 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u
                 }
                 echo (isset($logs) && !empty($logs)) ? json_encode($logs, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT) : '{}';
               break;
+              case "ratelimited":
+                // 0 is first record, so empty is fine
+                if (isset($extra)) {
+                  $extra = preg_replace('/[^\d\-]/i', '', $extra);
+                  $logs = get_logs('ratelimited', $extra);
+                }
+                else {
+                  $logs = get_logs('ratelimited');
+                }
+                echo (isset($logs) && !empty($logs)) ? json_encode($logs, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT) : '{}';
+              break;
               case "netfilter":
                 // 0 is first record, so empty is fine
                 if (isset($extra)) {
@@ -1043,6 +1054,9 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u
           case "admin":
             process_delete_return(admin('delete', array('username' => $items)));
           break;
+          case "rlhash":
+            echo ratelimit('delete', null, implode($items));
+          break;
         }
       break;
       case "edit":

data/web/lang/lang.de.php

@@ -556,10 +556,18 @@ $lang['admin']['no_record'] = 'Kein Eintrag';
 $lang['admin']['filter_table'] = 'Tabelle Filtern';
 $lang['admin']['empty'] = 'Keine Einträge vorhanden';
 $lang['admin']['time'] = 'Zeit';
+$lang['admin']['last_applied'] = 'Zuletzt angewendet';
+$lang['admin']['reset_limit'] = 'Hash entfernen';
+$lang['admin']['hash_remove_info'] = 'Das Entfernen eines Ratelimit Hashes - sofern noch existent - bewirkt den Reset gezählter Nachrichten dieses Elements.<br>
+  Jeder Hash wird durch eine eindeutige Farbe gekennzeichnet.';
+$lang['warning']['hash_not_found'] = 'Hash nicht gefunden';
+$lang['success']['hash_deleted'] = 'Hash wurde gelöscht';
+$lang['admin']['authed_user'] = 'Auth. Benutzer';
 $lang['admin']['priority'] = 'Gewichtung';
 $lang['admin']['refresh'] = 'Neu laden';
 $lang['admin']['to_top'] = 'Nach oben';
 $lang['admin']['in_use_by'] = 'Verwendet von';
+$lang['admin']['rate_name'] = 'Rate name';
 $lang['admin']['message'] = 'Nachricht';
 $lang['admin']['forwarding_hosts'] = 'Weiterleitungs-Hosts';
 $lang['admin']['forwarding_hosts_hint'] = 'Eingehende Nachrichten werden von den hier gelisteten Hosts bedingungslos akzeptiert. Diese Hosts werden dann nicht mit DNSBLs abgeglichen oder Greylisting unterworfen. Von ihnen empfangener Spam wird nie abgelehnt, optional kann er aber in den Spam-Ordner einsortiert werden. Die übliche Verwendung für diese Funktion ist, um Mailserver anzugeben, auf denen eine Weiterleitung zu Ihrem mailcow-Server eingerichtet wurde.';

data/web/lang/lang.en.php

@@ -580,8 +580,16 @@ $lang['admin']['no_record'] = 'No record';
 $lang['admin']['filter_table'] = 'Filter table';
 $lang['admin']['empty'] = 'No results';
 $lang['admin']['time'] = 'Time';
+$lang['admin']['last_applied'] = 'Last applied';
+$lang['admin']['reset_limit'] = 'Remove hash';
+$lang['admin']['hash_remove_info'] = 'Removing a ratelimit hash (if still existing) will reset its counter completely.<br>
+  Each hash is indicated by an individual color.';
+$lang['warning']['hash_not_found'] = 'Hash not found';
+$lang['success']['hash_deleted'] = 'Hash deleted';
+$lang['admin']['authed_user'] = 'Auth. user';
 $lang['admin']['priority'] = 'Priority';
 $lang['admin']['message'] = 'Message';
+$lang['admin']['rate_name'] = 'Rate name';
 $lang['admin']['refresh'] = 'Refresh';
 $lang['admin']['to_top'] = 'Back to top';
 $lang['admin']['in_use_by'] = 'In use by';