diff --git a/data/Dockerfiles/rspamd/Dockerfile b/data/Dockerfiles/rspamd/Dockerfile index e5172bf0..ded516eb 100644 --- a/data/Dockerfiles/rspamd/Dockerfile +++ b/data/Dockerfiles/rspamd/Dockerfile @@ -22,9 +22,7 @@ RUN apt-get update && apt-get install -y \ COPY settings.conf /etc/rspamd/modules.d/settings.conf COPY ratelimit.lua /usr/share/rspamd/lua/ratelimit.lua -#COPY lua_util.lua /usr/share/rspamd/lib/lua_util.lua COPY docker-entrypoint.sh /docker-entrypoint.sh -COPY tini /sbin/tini ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/data/Dockerfiles/rspamd/docker-entrypoint.sh b/data/Dockerfiles/rspamd/docker-entrypoint.sh index 15ae73da..afb03bb6 100755 --- a/data/Dockerfiles/rspamd/docker-entrypoint.sh +++ b/data/Dockerfiles/rspamd/docker-entrypoint.sh @@ -3,4 +3,4 @@ chown -R _rspamd:_rspamd /var/lib/rspamd [[ ! -f /etc/rspamd/override.d/worker-controller-password.inc ]] && echo '# Placeholder' > /etc/rspamd/override.d/worker-controller-password.inc -exec /sbin/tini -- "$@" +exec "$@" diff --git a/data/Dockerfiles/rspamd/ratelimit.lua b/data/Dockerfiles/rspamd/ratelimit.lua index e25ea42d..839ec5c6 100644 --- a/data/Dockerfiles/rspamd/ratelimit.lua +++ b/data/Dockerfiles/rspamd/ratelimit.lua @@ -21,26 +21,130 @@ end -- A plugin that implements ratelimits using redis -local E, settings = {}, {} +local E = {} local N = 'ratelimit' --- Senders that are considered as bounce -local bounce_senders = {'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon'} --- Do not check ratelimits for these recipients -local whitelisted_rcpts = {'postmaster', 'mailer-daemon'} -local whitelisted_ip -local whitelisted_user -local max_rcpt = 5 local redis_params -local ratelimit_symbol --- Do not delay mail after 1 day -local use_ip_score = false -local rl_prefix = 'RL' -local ip_score_lower_bound = 10 -local ip_score_ham_multiplier = 1.1 -local ip_score_spam_divisor = 1.1 -local limits_hash +-- Senders that are considered as bounce +local settings = { + bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' }, +-- Do not check ratelimits for these recipients + whitelisted_rcpts = { 'postmaster', 'mailer-daemon' }, + prefix = 'RL', + ham_factor_rate = 1.01, + spam_factor_rate = 0.99, + ham_factor_burst = 1.02, + spam_factor_burst = 0.98, + max_rate_mult = 5, + max_bucket_mult = 10, + expire = 60 * 60 * 24 * 2, -- 2 days by default + limits = {}, + allow_local = false, +} -local message_func = function(_, limit_type) +-- Checks bucket, updating it if needed +-- KEYS[1] - prefix to update, e.g. RL__ +-- KEYS[2] - current time in milliseconds +-- KEYS[3] - bucket leak rate (messages per millisecond) +-- KEYS[4] - bucket burst +-- KEYS[5] - expire for a bucket +-- return 1 if message should be ratelimited and 0 if not +-- Redis keys used: +-- l - last hit +-- b - current burst +-- dr - current dynamic rate multiplier (*10000) +-- db - current dynamic burst multiplier (*10000) +local bucket_check_script = [[ + local last = redis.call('HGET', KEYS[1], 'l') + local now = tonumber(KEYS[2]) + local dynr, dynb = 0, 0 + if not last then + -- New bucket + redis.call('HSET', KEYS[1], 'l', KEYS[2]) + redis.call('HSET', KEYS[1], 'b', '0') + redis.call('HSET', KEYS[1], 'dr', '10000') + redis.call('HSET', KEYS[1], 'db', '10000') + redis.call('EXPIRE', KEYS[1], KEYS[5]) + return {0, 0, 1, 1} + end + + last = tonumber(last) + local burst = tonumber(redis.call('HGET', KEYS[1], 'b')) + -- Perform leak + if burst > 0 then + if last < tonumber(KEYS[2]) then + local rate = tonumber(KEYS[3]) + dynr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000.0 + rate = rate * dynr + local leaked = ((now - last) * rate) + burst = burst - leaked + redis.call('HINCRBYFLOAT', KEYS[1], 'b', -(leaked)) + end + else + burst = 0 + redis.call('HSET', KEYS[1], 'b', '0') + end + + dynb = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000.0 + + if (burst + 1) * dynb > tonumber(KEYS[4]) then + return {1, tostring(burst), tostring(dynr), tostring(dynb)} + end + + return {0, tostring(burst), tostring(dynr), tostring(dynb)} +]] +local bucket_check_id + + +-- Updates a bucket +-- KEYS[1] - prefix to update, e.g. RL__ +-- KEYS[2] - current time in milliseconds +-- KEYS[3] - dynamic rate multiplier +-- KEYS[4] - dynamic burst multiplier +-- KEYS[5] - max dyn rate (min: 1/x) +-- KEYS[6] - max burst rate (min: 1/x) +-- KEYS[7] - expire for a bucket +-- Redis keys used: +-- l - last hit +-- b - current burst +-- dr - current dynamic rate multiplier +-- db - current dynamic burst multiplier +local bucket_update_script = [[ + local last = redis.call('HGET', KEYS[1], 'l') + local now = tonumber(KEYS[2]) + if not last then + -- New bucket + redis.call('HSET', KEYS[1], 'l', KEYS[2]) + redis.call('HSET', KEYS[1], 'b', '1') + redis.call('HSET', KEYS[1], 'dr', '10000') + redis.call('HSET', KEYS[1], 'db', '10000') + redis.call('EXPIRE', KEYS[1], KEYS[7]) + return {1, 1, 1} + end + + local burst = tonumber(redis.call('HGET', KEYS[1], 'b')) + local db = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000 + local dr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000 + + if dr < tonumber(KEYS[5]) and dr > 1.0 / tonumber(KEYS[5]) then + dr = dr * tonumber(KEYS[3]) + redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000))) + end + + if db < tonumber(KEYS[6]) and db > 1.0 / tonumber(KEYS[6]) then + db = db * tonumber(KEYS[4]) + redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000))) + end + + redis.call('HINCRBYFLOAT', KEYS[1], 'b', 1) + redis.call('HSET', KEYS[1], 'l', KEYS[2]) + redis.call('EXPIRE', KEYS[1], KEYS[7]) + + return {tostring(burst), tostring(dr), tostring(db)} +]] +local bucket_update_id + +-- message_func(task, limit_type, prefix, bucket) +local message_func = function(_, limit_type, _, _) return string.format('Ratelimit "%s" exceeded', limit_type) end @@ -49,137 +153,14 @@ local rspamd_util = require "rspamd_util" local rspamd_lua_utils = require "lua_util" local lua_redis = require "lua_redis" local fun = require "fun" +local lua_maps = require "lua_maps" +local lua_util = require "lua_util" +local rspamd_hash = require "rspamd_cryptobox_hash" -local user_keywords = {'user'} - -local redis_script_sha -local redis_script = [[local bucket -local limited = false -local buckets = {} -local queue_id = table.remove(ARGV) -local now = table.remove(ARGV) - -local argi = 0 -for i = 1, #KEYS do - local key = KEYS[i] - local period = tonumber(ARGV[argi+1]) - local limit = tonumber(ARGV[argi+2]) - if not buckets[key] then - buckets[key] = { - max_period = period, - limits = { {period, limit} }, - } - else - table.insert(buckets[key].limits, {period, limit}) - if period > buckets[key].max_period then - buckets[key].max_period = period - end - end - argi = argi + 2 -end - -for k, v in pairs(buckets) do - local maxp = v.max_period - redis.call('ZREMRANGEBYSCORE', k, '-inf', now - maxp) - for _, lim in ipairs(v.limits) do - local period = lim[1] - local limit = lim[2] - local rate - if period == maxp then - rate = redis.call('ZCARD', k) - else - rate = redis.call('ZCOUNT', k, now - period, '+inf') - end - if rate and rate >= limit then - limited = true - bucket = k - end - end - redis.call('EXPIRE', k, maxp) - if limited then break end -end - -if not limited then - for k in pairs(buckets) do - redis.call('ZADD', k, now, queue_id) - end -end - -return {limited, bucket}]] - -local redis_script_symbol = [[local limited = false -local buckets, results = {}, {} -local queue_id = table.remove(ARGV) -local now = table.remove(ARGV) - -local argi = 0 -for i = 1, #KEYS do - local key = KEYS[i] - local period = tonumber(ARGV[argi+1]) - local limit = tonumber(ARGV[argi+2]) - if not buckets[key] then - buckets[key] = { - max_period = period, - limits = { {period, limit} }, - } - else - table.insert(buckets[key].limits, {period, limit}) - if period > buckets[key].max_period then - buckets[key].max_period = period - end - end - argi = argi + 2 -end - -for k, v in pairs(buckets) do - local maxp = v.max_period - redis.call('ZREMRANGEBYSCORE', k, '-inf', now - maxp) - for _, lim in ipairs(v.limits) do - local period = lim[1] - local limit = lim[2] - local rate - if period == maxp then - rate = redis.call('ZCARD', k) - else - rate = redis.call('ZCOUNT', k, now - period, '+inf') - end - if rate then - local mult = 2 * math.tanh(rate / (limit * 2)) - if mult >= 0.5 then - table.insert(results, {k, tostring(mult)}) - end - end - end - redis.call('ZADD', k, now, queue_id) - redis.call('EXPIRE', k, maxp) -end - -return results]] local function load_scripts(cfg, ev_base) - local function rl_script_cb(err, data) - if err then - rspamd_logger.errx(cfg, 'Script loading failed: ' .. err) - elseif type(data) == 'string' then - redis_script_sha = data - end - end - local script - if ratelimit_symbol then - script = redis_script_symbol - else - script = redis_script - end - lua_redis.redis_make_request_taskless( - ev_base, - cfg, - redis_params, - nil, -- key - true, -- is write - rl_script_cb, --callback - 'SCRIPT', -- command - {'LOAD', script} - ) + bucket_check_id = lua_redis.add_redis_script(bucket_check_script, redis_params) + bucket_update_id = lua_redis.add_redis_script(bucket_update_script, redis_params) end local limit_parser @@ -246,43 +227,67 @@ local function parse_string_limit(lim, no_error) return nil end -local function resize_element(x_score, x_total, element) - local x_ip_score - if not x_total then x_total = 0 end - if x_total < ip_score_lower_bound or x_total <= 0 then - x_score = 1 - else - x_score = x_score / x_total +local function parse_limit(name, data) + local buckets = {} + if type(data) == 'table' then + -- 3 cases here: + -- * old limit in format [burst, rate] + -- * vector of strings in Andrew's string format + -- * proper bucket table + if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then + -- Old style ratelimit + rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name) + if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then + table.insert(buckets, { + burst = data[1], + rate = data[2] + }) + elseif data[1] ~= 0 then + rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name) + else + rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name) + end + else + -- Recursively map parse_limit and flatten the list + fun.each(function(l) + -- Flatten list + for _,b in ipairs(l) do table.insert(buckets, b) end + end, fun.map(function(d) return parse_limit(d, name) end, data)) + end + elseif type(data) == 'string' then + local rep_rate, burst = parse_string_limit(data) + + if rep_rate and burst then + table.insert(buckets, { + burst = burst, + rate = 1.0 / rep_rate -- reciprocal + }) + end end - if x_score > 0 then - x_ip_score = x_score / ip_score_spam_divisor - element = element * rspamd_util.tanh(2.718281 * x_ip_score) - elseif x_score < 0 then - x_ip_score = ((1 + (x_score * -1)) * ip_score_ham_multiplier) - element = element * x_ip_score - end - return element + + -- Filter valid + return fun.totable(fun.filter(function(val) + return type(val.burst) == 'number' and type(val.rate) == 'number' + end, buckets)) end --- Check whether this addr is bounce local function check_bounce(from) - return fun.any(function(b) return b == from end, bounce_senders) + return fun.any(function(b) return b == from end, settings.bounce_senders) end -local custom_keywords = {} - local keywords = { ['ip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() then return ip end + if ip and ip:is_valid() then return tostring(ip) end return nil end, }, ['rip'] = { ['get_value'] = function(task) local ip = task:get_ip() - if ip and ip:is_valid() and not ip:is_local() then return ip end + if ip and ip:is_valid() and not ip:is_local() then return tostring(ip) end return nil end, }, @@ -325,144 +330,84 @@ local keywords = { end, }, ['to'] = { - ['get_value'] = function() - return '%s' -- 'to' is special + ['get_value'] = function(task) + return task:get_principal_recipient() end, }, } -local function dynamic_rate_key(task, rtype) - local key_t = {rl_prefix, rtype} - local key_keywords = rspamd_str_split(rtype, '_') - local have_to, have_user = false, false +local function gen_rate_key(task, rtype, bucket) + local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))} + local key_keywords = lua_util.str_split(rtype, '_') + local have_user = false + for _, v in ipairs(key_keywords) do - if (custom_keywords[v] and type(custom_keywords[v]['condition']) == 'function') then - if not custom_keywords[v]['condition']() then return nil end - end local ret - if custom_keywords[v] and type(custom_keywords[v]['get_value']) == 'function' then - ret = custom_keywords[v]['get_value'](task) - elseif keywords[v] and type(keywords[v]['get_value']) == 'function' then + + if keywords[v] and type(keywords[v]['get_value']) == 'function' then ret = keywords[v]['get_value'](task) end if not ret then return nil end - for _, uk in ipairs(user_keywords) do - if v == uk then have_user = true end - if have_user then break end - end - if v == 'to' then have_to = true end + if v == 'user' then have_user = true end if type(ret) ~= 'string' then ret = tostring(ret) end table.insert(key_t, ret) end - if (not have_user) and task:get_user() then + + if have_user and not task:get_user() then return nil end - if not have_to then - return table.concat(key_t, ":") - else - local rate_keys = {} - local rcpts = task:get_recipients(0) - if not ((rcpts or E)[1] or E).addr then - return nil - end - local key_s = table.concat(key_t, ":") - local total_rcpt = 0 - for _, r in ipairs(rcpts) do - if r['addr'] and total_rcpt < max_rcpt then - local key_f = string.format(key_s, string.lower(r['addr'])) - table.insert(rate_keys, key_f) - total_rcpt = total_rcpt + 1 - end - end - return rate_keys - end + + return table.concat(key_t, ":") end -local function process_buckets(task, buckets) - if not buckets then return end - local function rl_redis_cb(err, data) - if err then - rspamd_logger.infox(task, 'got error while setting limit: %1', err) - end - if not data then return end - if data[1] == 1 then - rspamd_logger.infox(task, - 'ratelimit "%s" exceeded', - data[2]) - task:set_pre_result('soft reject', - message_func(task, data[2])) +local function make_prefix(redis_key, name, bucket) + local hash_len = 24 + if hash_len > #redis_key then hash_len = #redis_key end + local hash = settings.prefix .. + string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len) + -- Fill defaults + if not bucket.spam_factor_rate then + bucket.spam_factor_rate = settings.spam_factor_rate + end + if not bucket.ham_factor_rate then + bucket.ham_factor_rate = settings.ham_factor_rate + end + if not bucket.spam_factor_burst then + bucket.spam_factor_burst = settings.spam_factor_burst + end + if not bucket.ham_factor_burst then + bucket.ham_factor_burst = settings.ham_factor_burst + end + + return { + bucket = bucket, + name = name, + hash = hash + } +end + +local function limit_to_prefixes(task, k, v, prefixes) + local n = 0 + for _,bucket in ipairs(v) do + local prefix = gen_rate_key(task, k, bucket) + + if prefix then + prefixes[prefix] = make_prefix(prefix, k, bucket) + n = n + 1 end end - local function rl_symbol_redis_cb(err, data) - if err then - rspamd_logger.infox(task, 'got error while setting limit: %1', err) - end - if not data then return end - for i, b in ipairs(data) do - task:insert_result(ratelimit_symbol, b[2], string.format('%s:%s:%s', i, b[1], b[2])) - end - end - local redis_cb = rl_redis_cb - if ratelimit_symbol then redis_cb = rl_symbol_redis_cb end - local args = {redis_script_sha, #buckets} - for _, bucket in ipairs(buckets) do - table.insert(args, bucket[2]) - end - for _, bucket in ipairs(buckets) do - if use_ip_score then - local asn_score,total_asn, - country_score,total_country, - ipnet_score,total_ipnet, - ip_score, total_ip = task:get_mempool():get_variable('ip_score', - 'double,double,double,double,double,double,double,double') - local key_keywords = rspamd_str_split(bucket[2], '_') - local has_asn, has_ip = false, false - for _, v in ipairs(key_keywords) do - if v == "asn" then has_asn = true end - if v == "ip" then has_ip = true end - if has_ip and has_asn then break end - end - if has_asn and not has_ip then - bucket[1][2] = resize_element(asn_score, total_asn, bucket[1][2]) - elseif has_ip then - if total_ip and total_ip > ip_score_lower_bound then - bucket[1][2] = resize_element(ip_score, total_ip, bucket[1][2]) - elseif total_ipnet and total_ipnet > ip_score_lower_bound then - bucket[1][2] = resize_element(ipnet_score, total_ipnet, bucket[1][2]) - elseif total_asn and total_asn > ip_score_lower_bound then - bucket[1][2] = resize_element(asn_score, total_asn, bucket[1][2]) - elseif total_country and total_country > ip_score_lower_bound then - bucket[1][2] = resize_element(country_score, total_country, bucket[1][2]) - else - bucket[1][2] = resize_element(ip_score, total_ip, bucket[1][2]) - end - end - end - table.insert(args, bucket[1][1]) - table.insert(args, bucket[1][2]) - end - table.insert(args, rspamd_util.get_time()) - table.insert(args, task:get_queue_id() or task:get_uid()) - local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - redis_cb, --callback - 'evalsha', -- command - args -- arguments - ) - if not ret then - rspamd_logger.errx(task, 'got error connecting to redis') - end + + return n end local function ratelimit_cb(task) - if rspamd_lua_utils.is_rspamc_or_controller(task) then return end - local args = {} + if not settings.allow_local and + rspamd_lua_utils.is_rspamc_or_controller(task) then return end + -- Get initial task data local ip = task:get_from_ip() - if ip and ip:is_valid() and whitelisted_ip then - if whitelisted_ip:get_key(ip) then + if ip and ip:is_valid() and settings.whitelisted_ip then + if settings.whitelisted_ip:get_key(ip) then -- Do not check whitelisted ip rspamd_logger.infox(task, 'skip ratelimit for whitelisted IP') return @@ -475,136 +420,149 @@ local function ratelimit_cb(task) fun.each(function(r) fun.each(function(type) table.insert(rcpts_user, r[type]) end, {'user', 'addr'}) end, rcpts) - if fun.any( - function(r) - if fun.any(function(w) return r == w end, whitelisted_rcpts) then return true end - end, - rcpts_user) then + if fun.any(function(r) return settings.whitelisted_rcpts:get_key(r) end, rcpts_user) then rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient') return end end -- Get user (authuser) - if whitelisted_user then + if settings.whitelisted_user then local auser = task:get_user() - if whitelisted_user:get_key(auser) then + if settings.whitelisted_user:get_key(auser) then rspamd_logger.infox(task, 'skip ratelimit for whitelisted user') return end end + -- Now create all ratelimit prefixes + local prefixes = {} + local nprefixes = 0 - local redis_keys = {} - local redis_keys_rev = {} - local function collect_redis_keys() - local function collect_cb(err, data) - if err then - rspamd_logger.errx(task, 'redis error: %1', err) - else - for i, d in ipairs(data) do - if type(d) == 'string' then - local plim, size = parse_string_limit(d) - if plim then - table.insert(args, {{plim, size}, redis_keys_rev[i]}) - end - end - end - return process_buckets(task, args) + for k,v in pairs(settings.limits) do + nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes) + end + + for k, hdl in pairs(settings.custom_keywords or E) do + local ret, redis_key, bd = pcall(hdl, task) + + if ret then + local bucket = parse_limit(k, bd) + if bucket[1] then + prefixes[redis_key] = make_prefix(redis_key, k, bucket[1]) end - end - local params, method - if limits_hash then - params = {limits_hash, rspamd_lua_utils.unpack(redis_keys)} - method = 'HMGET' + nprefixes = nprefixes + 1 else - method = 'MGET' - params = redis_keys - end - local requested_keys = rspamd_redis_make_request(task, - redis_params, -- connect params - nil, -- hash key - true, -- is write - collect_cb, --callback - method, -- command - params -- arguments - ) - if not requested_keys then - rspamd_logger.errx(task, 'got error connecting to redis') - return process_buckets(task, args) + rspamd_logger.errx(task, 'cannot call handler for %s: %s', + k, redis_key) end end - local rate_key - for k in pairs(settings) do - rate_key = dynamic_rate_key(task, k) - if rate_key then - if type(rate_key) == 'table' then - for _, rk in ipairs(rate_key) do - if type(settings[k]) == 'string' and - (custom_keywords[settings[k]] and type(custom_keywords[settings[k]]['get_limit']) == 'function') then - local res = custom_keywords[settings[k]]['get_limit'](task) - if type(res) == 'string' then res = {res} end - for _, r in ipairs(res) do - local plim, size = parse_string_limit(r, true) - if plim then - table.insert(args, {{plim, size}, rk}) - else - local rkey = string.match(settings[k], 'redis:(.*)') - if rkey then - table.insert(redis_keys, rkey) - redis_keys_rev[#redis_keys] = rk - else - rspamd_logger.infox(task, "Don't know what to do with limit: %1", settings[k]) - end - end - end - end - end - else - if type(settings[k]) == 'string' and - (custom_keywords[settings[k]] and type(custom_keywords[settings[k]]['get_limit']) == 'function') then - local res = custom_keywords[settings[k]]['get_limit'](task) - if type(res) == 'string' then res = {res} end - for _, r in ipairs(res) do - local plim, size = parse_string_limit(r, true) - if plim then - table.insert(args, {{plim, size}, rate_key}) - else - local rkey = string.match(r, 'redis:(.*)') - if rkey then - table.insert(redis_keys, rkey) - redis_keys_rev[#redis_keys] = rate_key - else - rspamd_logger.infox(task, "Don't know what to do with limit: %1", settings[k]) - end - end - end - elseif type(settings[k]) == 'table' then - for _, rl in ipairs(settings[k]) do - table.insert(args, {{rl[1], rl[2]}, rate_key}) - end - elseif type(settings[k]) == 'string' then - local rkey = string.match(settings[k], 'redis:(.*)') - if rkey then - table.insert(redis_keys, rkey) - redis_keys_rev[#redis_keys] = rate_key - else - rspamd_logger.infox(task, "Don't know what to do with limit: %1", settings[k]) - end + local function gen_check_cb(prefix, bucket, lim_name) + return function(err, data) + if err then + rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data) + elseif type(data) == 'table' and data[1] and data[1] == 1 then + -- set symbol only and do NOT soft reject + if settings.symbol then + task:insert_result(settings.symbol, 0.0, lim_name .. "(" .. prefix .. ")") + rspamd_logger.infox(task, + 'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)', + lim_name, prefix, + bucket.burst, bucket.rate, + data[2], data[3], data[4]) + return + -- set INFO symbol and soft reject + elseif settings.info_symbol then + task:insert_result(settings.info_symbol, 1.0, + lim_name .. "(" .. prefix .. ")") end + rspamd_logger.infox(task, + 'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)', + lim_name, prefix, + bucket.burst, bucket.rate, + data[2], data[3], data[4]) + task:set_pre_result('soft reject', + message_func(task, lim_name, prefix, bucket)) end end end - if redis_keys[1] then - return collect_redis_keys() - else - return process_buckets(task, args) + -- Don't do anything if pre-result has been already set + if task:has_pre_result() then return end + + if nprefixes > 0 then + -- Save prefixes to the cache to allow update + task:cache_set('ratelimit_prefixes', prefixes) + local now = rspamd_util.get_time() + now = lua_util.round(now * 1000.0) -- Get milliseconds + -- Now call check script for all defined prefixes + + for pr,value in pairs(prefixes) do + local bucket = value.bucket + local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms + rspamd_logger.debugm(N, task, "check limit %s:%s -> %s (%s/%s)", + value.name, pr, value.hash, bucket.burst, bucket.rate) + lua_redis.exec_redis_script(bucket_check_id, + {key = value.hash, task = task, is_write = true}, + gen_check_cb(pr, bucket, value.name), + {value.hash, tostring(now), tostring(rate), tostring(bucket.burst), + tostring(settings.expire)}) + end + end +end + +local function ratelimit_update_cb(task) + local prefixes = task:cache_get('ratelimit_prefixes') + + if prefixes then + if task:has_pre_result() then + -- Already rate limited/greylisted, do nothing + rspamd_logger.debugm(N, task, 'pre-action has been set, do not update') + return + end + + local is_spam = not (task:get_metric_action() == 'no action') + + -- Update each bucket + for k, v in pairs(prefixes) do + local bucket = v.bucket + local function update_bucket_cb(err, data) + if err then + rspamd_logger.errx(task, 'cannot update rate bucket %s: %s', + k, err) + else + rspamd_logger.debugm(N, task, + "updated limit %s:%s -> %s (%s/%s), burst: %s, dyn_rate: %s, dyn_burst: %s", + v.name, k, v.hash, + bucket.burst, bucket.rate, + data[1], data[2], data[3]) + end + end + local now = rspamd_util.get_time() + now = lua_util.round(now * 1000.0) -- Get milliseconds + local mult_burst = bucket.ham_factor_burst or 1.0 + local mult_rate = bucket.ham_factor_burst or 1.0 + + if is_spam then + mult_burst = bucket.spam_factor_burst or 1.0 + mult_rate = bucket.spam_factor_rate or 1.0 + end + + lua_redis.exec_redis_script(bucket_update_id, + {key = v.hash, task = task, is_write = true}, + update_bucket_cb, + {v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst), + tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult), + tostring(settings.expire)}) + end end end local opts = rspamd_config:get_all_opt(N) if opts then + + settings = lua_util.override_defaults(settings, opts) + if opts['limit'] then rspamd_logger.errx(rspamd_config, 'Legacy ratelimit config format no longer supported') end @@ -612,112 +570,105 @@ if opts then if opts['rates'] and type(opts['rates']) == 'table' then -- new way of setting limits fun.each(function(t, lim) - if type(lim) == 'table' then - settings[t] = {} - fun.each(function(l) - local plim, size = parse_string_limit(l) - if plim then - table.insert(settings[t], {plim, size}) - end - end, lim) - elseif type(lim) == 'string' then - local plim, size = parse_string_limit(lim) - if plim then - settings[t] = { {plim, size} } - end + local buckets = parse_limit(t, lim) + + if buckets and #buckets > 0 then + settings.limits[t] = buckets end end, opts['rates']) end - if opts['dynamic_rates'] and type(opts['dynamic_rates']) == 'table' then - fun.each(function(t, lim) - if type(lim) == 'string' then - settings[t] = lim - end - end, opts['dynamic_rates']) - end - local enabled_limits = fun.totable(fun.map(function(t) return t - end, settings)) - rspamd_logger.infox(rspamd_config, 'enabled rate buckets: [%1]', table.concat(enabled_limits, ',')) + end, settings.limits)) + rspamd_logger.infox(rspamd_config, + 'enabled rate buckets: [%1]', table.concat(enabled_limits, ',')) - if opts['whitelisted_rcpts'] and type(opts['whitelisted_rcpts']) == 'string' then - whitelisted_rcpts = rspamd_str_split(opts['whitelisted_rcpts'], ',') + -- Ret, ret, ret: stupid legacy stuff: + -- If we have a string with commas then load it as as static map + -- otherwise, apply normal logic of Rspamd maps + + local wrcpts = opts['whitelisted_rcpts'] + if type(wrcpts) == 'string' then + if string.find(wrcpts, ',') then + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( + lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts') + else + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', + 'Ratelimit whitelisted rcpts') + end elseif type(opts['whitelisted_rcpts']) == 'table' then - whitelisted_rcpts = opts['whitelisted_rcpts'] + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', + 'Ratelimit whitelisted rcpts') + else + -- Stupid default... + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( + settings.whitelisted_rcpts, 'set', 'Ratelimit whitelisted rcpts') end if opts['whitelisted_ip'] then - whitelisted_ip = rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix', + settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix', 'Ratelimit whitelist ip map') end if opts['whitelisted_user'] then - whitelisted_user = rspamd_map_add('ratelimit', 'whitelisted_user', 'set', + settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set', 'Ratelimit whitelist user map') end - if opts['symbol'] then - -- We want symbol instead of pre-result - ratelimit_symbol = opts['symbol'] - end - - if opts['max_rcpt'] then - max_rcpt = tonumber(opts['max_rcpt']) - end - - if opts['use_ip_score'] then - use_ip_score = true - local ip_score_opts = rspamd_config:get_all_opt('ip_score') - if ip_score_opts and ip_score_opts['lower_bound'] then - ip_score_lower_bound = ip_score_opts['lower_bound'] - end - end - + settings.custom_keywords = {} if opts['custom_keywords'] then - custom_keywords = dofile(opts['custom_keywords']) - end + local ret, res_or_err = pcall(loadfile(opts['custom_keywords'])) - if opts['user_keywords'] then - user_keywords = opts['user_keywords'] + if ret then + opts['custom_keywords'] = {} + if type(res_or_err) == 'table' then + for k,hdl in pairs(res_or_err) do + settings['custom_keywords'][k] = hdl + end + elseif type(res_or_err) == 'function' then + settings['custom_keywords']['custom'] = res_or_err + end + else + rspamd_logger.errx(rspamd_config, 'cannot execute %s: %s', + opts['custom_keywords'], res_or_err) + settings['custom_keywords'] = {} + end end if opts['message_func'] then message_func = assert(load(opts['message_func']))() end - if opts['limits_hash'] then - limits_hash = opts['limits_hash'] - end + redis_params = lua_redis.parse_redis_server('ratelimit') - redis_params = rspamd_parse_redis_server('ratelimit') if not redis_params then rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module') + lua_util.disable_module(N, "redis") else local s = { type = 'prefilter,nostat', name = 'RATELIMIT_CHECK', - priority = 4, + priority = 7, callback = ratelimit_cb, + flags = 'empty', } - if use_ip_score then - s.type = 'normal' - end - if ratelimit_symbol then - s.name = ratelimit_symbol - end - local id = rspamd_config:register_symbol(s) - if use_ip_score then - rspamd_config:register_dependency(id, 'IP_SCORE') - end - for _, v in pairs(custom_keywords) do - if type(v) == 'table' and type(v['init']) == 'function' then - v['init']() - end + + if settings.symbol then + s.name = settings.symbol + elseif settings.info_symbol then + s.name = settings.info_symbol end + + rspamd_config:register_symbol(s) + rspamd_config:register_symbol { + type = 'idempotent', + name = 'RATELIMIT_UPDATE', + callback = ratelimit_update_cb, + } end end + rspamd_config:add_on_load(function(cfg, ev_base, worker) load_scripts(cfg, ev_base) end) diff --git a/data/Dockerfiles/rspamd/tini b/data/Dockerfiles/rspamd/tini deleted file mode 100755 index 6556c966..00000000 Binary files a/data/Dockerfiles/rspamd/tini and /dev/null differ diff --git a/data/web/admin.php b/data/web/admin.php index 81b36ba8..ce3deb69 100644 --- a/data/web/admin.php +++ b/data/web/admin.php @@ -657,7 +657,7 @@ $tfa_data = get_tfa();

- + diff --git a/data/web/css/admin.css b/data/web/css/admin.css index 880bb6bd..bd6ce1a5 100644 --- a/data/web/css/admin.css +++ b/data/web/css/admin.css @@ -64,4 +64,10 @@ body.modal-open { font-family:Consolas,Monaco,Lucida Console,Liberation Mono,DejaVu Sans Mono,Bitstream Vera Sans Mono,Courier New, monospace; font-size:9pt; background:transparent; -} \ No newline at end of file +} +.bootstrap-select { + width: auto!important; +} +.table-condensed .input-sm { + width: 100%!important; +} diff --git a/data/web/js/admin.js b/data/web/js/admin.js index 72a5c3bb..8c9876c0 100644 --- a/data/web/js/admin.js +++ b/data/web/js/admin.js @@ -79,7 +79,7 @@ jQuery(function($){ {"name":"id","type":"text","title":"ID","style":{"width":"50px"}}, {"name":"hostname","type":"text","title":lang.host,"style":{"width":"250px"}}, {"name":"username","title":lang.username,"breakpoints":"xs sm"}, - {"name":"used_by_domains","title":lang.in_use_by, "type": "text","breakpoints":"xs sm"}, + {"name":"used_by_domains","title":lang.in_use_by,"style":{"width":"110px"}, "type": "text","breakpoints":"xs sm"}, {"name":"active","filterable": false,"style":{"maxWidth":"80px","width":"80px"},"title":lang.active}, {"name":"action","filterable": false,"sortable": false,"style":{"text-align":"right","maxWidth":"280px","width":"280px"},"type":"html","title":lang.action,"breakpoints":"xs sm"} ], diff --git a/data/web/js/mailbox.js b/data/web/js/mailbox.js index 886c28ca..33d5519d 100644 --- a/data/web/js/mailbox.js +++ b/data/web/js/mailbox.js @@ -168,7 +168,7 @@ jQuery(function($){ return Number(res[0]); }, }, - {"name":"max_quota_for_mbox","title":lang.mailbox_quota,"breakpoints":"xs sm"}, + {"name":"max_quota_for_mbox","title":lang.mailbox_quota,"breakpoints":"xs sm","style":{"width":"125px"}}, {"name":"backupmx","filterable": false,"style":{"maxWidth":"120px","width":"120px"},"title":lang.backup_mx,"breakpoints":"xs sm"}, {"name":"active","filterable": false,"style":{"maxWidth":"80px","width":"80px"},"title":lang.active}, {"name":"action","filterable": false,"sortable": false,"style":{"text-align":"right","maxWidth":"240px","width":"240px"},"type":"html","title":lang.action,"breakpoints":"xs sm"} diff --git a/data/web/lang/lang.en.php b/data/web/lang/lang.en.php index d6a1c3ef..6ad8b27e 100644 --- a/data/web/lang/lang.en.php +++ b/data/web/lang/lang.en.php @@ -104,7 +104,7 @@ $lang['user']['new_password_description'] = 'Requirement: 6 characters long, let $lang['user']['spam_aliases'] = 'Temporary email aliases'; $lang['user']['alias'] = 'Alias'; $lang['user']['shared_aliases'] = 'Shared alias addresses'; -$lang['user']['shared_aliases_desc'] = 'A shared alias address is not affected by any user specific settings. A custom spam filter setting can be archived by a domain-wide policy set by an administrator..'; +$lang['user']['shared_aliases_desc'] = 'Shared aliases are not affected by user specific settings such as the spam filter or encryption policy. Corresponding spam filters can only be made by an administrator as a domain-wide policy..'; $lang['user']['direct_aliases'] = 'Direct alias addresses'; $lang['user']['direct_aliases_desc'] = 'Direct alias addresses are affected by spam filter and TLS policy settings.'; $lang['user']['is_catch_all'] = 'Catch-all for domain/s'; diff --git a/docker-compose.yml b/docker-compose.yml index cf747434..856efbc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,7 +78,7 @@ services: - clamd rspamd-mailcow: - image: mailcow/rspamd:1.22 + image: mailcow/rspamd:1.23 build: ./data/Dockerfiles/rspamd stop_grace_period: 30s depends_on: