/*
 * Copyright 2025 Vsevolod Stakhov
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * Rspamd fuzzy storage server
 */

#include "config.h"
#include "libserver/fuzzy_wire.h"
#include "libserver/fuzzy_storage_internal.h"
#include "util.h"
#include "rspamd.h"
#include "libserver/maps/map.h"
#include "libserver/maps/map_helpers.h"
#include "libserver/fuzzy_backend/fuzzy_backend.h"
#include "ottery.h"
#include "ref.h"
#include "xxhash.h"
#include "libserver/worker_util.h"
#include "libserver/rspamd_control.h"
#include "libcryptobox/cryptobox.h"
#include "libcryptobox/keypairs_cache.h"
#include "libcryptobox/keypair.h"
#include "libutil/hash.h"
#include "libserver/maps/map_private.h"
#include "contrib/uthash/utlist.h"
#include "lua/lua_common.h"
#include "unix-std.h"
#include "libutil/radix.h"

#include <math.h>

/* Resync value in seconds */
#define DEFAULT_SYNC_TIMEOUT 60.0
#define DEFAULT_KEYPAIR_CACHE_SIZE 512
#define DEFAULT_MASTER_TIMEOUT 10.0
#define DEFAULT_UPDATES_MAXFAIL 3
#define DEFAULT_MAX_BUCKETS 2000
#define DEFAULT_BUCKET_TTL 3600
#define DEFAULT_BUCKET_MASK 24
/* Update stats on keys each 1 hour */
#define KEY_STAT_INTERVAL 3600.0
/* TCP constants */
#define FUZZY_TCP_BUFFER_LENGTH 8192
#define DEFAULT_TCP_TIMEOUT 5.0

static const char *local_db_name = "local";

/* Init functions */
gpointer init_fuzzy(struct rspamd_config *cfg);
void start_fuzzy(struct rspamd_worker *worker);

#define msg_debug_fuzzy_storage(...) rspamd_conditional_debug_fast(NULL, NULL,                                         \
																   rspamd_fuzzy_storage_log_id, "fuzzy_storage", NULL, \
																   RSPAMD_LOG_FUNC,                                    \
																   __VA_ARGS__)
INIT_LOG_MODULE(fuzzy_storage)

worker_t fuzzy_worker = {
	"fuzzy",     /* Name */
	init_fuzzy,  /* Init function */
	start_fuzzy, /* Start function */
	RSPAMD_WORKER_HAS_SOCKET | RSPAMD_WORKER_NO_STRICT_CONFIG | RSPAMD_WORKER_FUZZY,
	RSPAMD_WORKER_SOCKET_UDP | RSPAMD_WORKER_SOCKET_TCP, /* UDP + TCP socket */
	RSPAMD_WORKER_VER                                    /* Version info */
};

static const uint64_t rspamd_fuzzy_storage_magic = 0x291a3253eb1b3ea5ULL;

struct rspamd_fuzzy_tcp_frame {
	uint16_t size_hdr; /* We have to write this as well */
	union {
		struct rspamd_fuzzy_encrypted_reply v1;
		struct rspamd_fuzzy_encrypted_reply_v2 v2;
	} payload; /* Payload - must be large enough for v2 replies */
};

struct fuzzy_tcp_reply_queue_elt {
	struct rspamd_fuzzy_tcp_frame rep;             /* Serialized reply */
	unsigned int written;                          /* How many bytes have we already written */
	struct fuzzy_tcp_reply_queue_elt *prev, *next; /* Link */
};

/*
 * Connection-level state for a TCP fuzzy session. Per-command state
 * (parsed cmd, key retain, extensions buffer, decrypt nm, ip_stat) lives
 * on the per-frame `struct fuzzy_session` and dies with it — never copy
 * it onto the long-lived TCP session.
 */
struct fuzzy_common_session {
	struct rspamd_fuzzy_storage_ctx *ctx;
	int fd;
	struct ev_io io;
	ev_tstamp timestamp;
	struct rspamd_worker *worker;
	rspamd_inet_addr_t *addr;
};

struct fuzzy_tcp_session {
	struct ev_timer tm;

	/*
	 * We store the state in the current frame
	 * 0 0 x x x x x x x x x x x x x x x - initial
	 * 1 0 x x x x x x x x x x x x x x x - read 1 byte of length
	 * 1 1 x x x x x x x x x x x x x x x - read 2 bytes of length
	 * So the length is always cur_frame & 0x3fff
	 */
	uint16_t cur_frame_state;
	uint16_t bytes_unprocessed;

	/*
	 * Set once by rspamd_fuzzy_tcp_session_close. The connection is gone,
	 * but the object can outlive it while in-flight commands still hold
	 * references, so every path that touches the socket or the watchers
	 * must check this first.
	 */
	bool closed;

	struct fuzzy_common_session common;
	ref_entry_t ref;

	struct fuzzy_tcp_reply_queue_elt *replies_queue;
	unsigned char input_buf[FUZZY_TCP_BUFFER_LENGTH];
};

struct fuzzy_peer_request {
	ev_io io_ev;
	struct fuzzy_peer_cmd cmd;
	/* ctx is set (and the request linked into ctx->pending_peer_requests) only
	 * while the write watcher is registered, so synchronous-success requests
	 * skip the list bookkeeping. */
	struct rspamd_fuzzy_storage_ctx *ctx;
	struct fuzzy_peer_request *prev;
	struct fuzzy_peer_request *next;
	gsize written; /* bytes of cmd already sent on the peer pipe */
};

enum fuzzy_peer_send_status {
	FUZZY_PEER_SEND_DONE = 0, /* fully sent */
	FUZZY_PEER_SEND_AGAIN,    /* short write or EAGAIN/EWOULDBLOCK/EINTR */
	FUZZY_PEER_SEND_FATAL,    /* unrecoverable write error */
};

enum fuzzy_tcp_write_status {
	FUZZY_TCP_WRITE_DONE = 0, /* reply fully sent and dequeued */
	FUZZY_TCP_WRITE_AGAIN,    /* short write or EAGAIN/EWOULDBLOCK/EINTR */
	FUZZY_TCP_WRITE_FATAL,    /* unrecoverable write error */
};

struct rspamd_updates_cbdata {
	GArray *updates_pending;
	struct rspamd_fuzzy_storage_ctx *ctx;
	char *source;
	gboolean final;
};

static void rspamd_fuzzy_write_reply(struct fuzzy_session *session);
static enum fuzzy_tcp_write_status rspamd_fuzzy_tcp_write_reply(struct fuzzy_tcp_session *session,
																struct fuzzy_tcp_reply_queue_elt *reply);
static void rspamd_fuzzy_tcp_session_close(struct fuzzy_tcp_session *session);
static gboolean rspamd_fuzzy_process_updates_queue(struct rspamd_fuzzy_storage_ctx *ctx,
												   const char *source, gboolean final);
static void rspamd_fuzzy_tcp_io(EV_P_ ev_io *w, int revents);
static void accept_tcp_socket(EV_P_ ev_io *w, int revents);

static gboolean
rspamd_fuzzy_check_write(struct rspamd_fuzzy_storage_ctx *ctx,
						 rspamd_inet_addr_t *addr,
						 struct fuzzy_key *key,
						 uint8_t cmd)
{
	if (ctx->read_only) {
		return FALSE;
	}

	/*
	 * Check IP first
	 */
	if (ctx->update_ips != NULL && addr) {
		if (rspamd_inet_address_get_af(addr) == AF_UNIX) {
			return TRUE;
		}
		if (rspamd_match_radix_map_addr(ctx->update_ips,
										addr) == NULL) {
			return FALSE;
		}
		else {
			return TRUE;
		}
	}

	/*
	 * Check global list of the update keys
	 */
	if (ctx->update_keys != NULL && key && key->stat && key->key) {
		static char base32_buf[rspamd_cryptobox_HASHBYTES * 2 + 1];
		unsigned int raw_len;
		const unsigned char *pk_raw = rspamd_keypair_component(key->key,
															   RSPAMD_KEYPAIR_COMPONENT_ID, &raw_len);
		int encoded_len = rspamd_encode_base32_buf(pk_raw, raw_len,
												   base32_buf, sizeof(base32_buf),
												   RSPAMD_BASE32_DEFAULT);

		if (rspamd_match_hash_map(ctx->update_keys, base32_buf, encoded_len)) {
			return TRUE;
		}
	}

	if (key) {
		if (cmd == FUZZY_WRITE && key->flags & FUZZY_KEY_WRITE) {
			return TRUE;
		}
		else if (cmd == FUZZY_DEL && key->flags & FUZZY_KEY_DELETE) {
			return TRUE;
		}
	}

	return FALSE;
}

static void
fuzzy_count_callback(uint64_t count, void *ud)
{
	struct rspamd_fuzzy_storage_ctx *ctx = ud;

	ctx->stat.fuzzy_hashes = count;
}

static void
fuzzy_stat_count_callback(uint64_t count, void *ud)
{
	struct rspamd_fuzzy_storage_ctx *ctx = ud;

	ev_timer_again(ctx->event_loop, &ctx->stat_ev);
	ctx->stat.fuzzy_hashes = count;
}

static void
rspamd_fuzzy_stat_callback(EV_P_ ev_timer *w, int revents)
{
	struct rspamd_fuzzy_storage_ctx *ctx =
		(struct rspamd_fuzzy_storage_ctx *) w->data;
	rspamd_fuzzy_backend_count(ctx->backend, fuzzy_stat_count_callback, ctx);
}


static void
fuzzy_update_version_callback(uint64_t ver, void *ud)
{
}

static void
rspamd_fuzzy_updates_cb(gboolean success,
						unsigned int nadded,
						unsigned int ndeleted,
						unsigned int nextended,
						unsigned int nignored,
						void *ud)
{
	struct rspamd_updates_cbdata *cbdata = ud;
	struct rspamd_fuzzy_storage_ctx *ctx;
	const char *source;

	ctx = cbdata->ctx;
	source = cbdata->source;

	if (success) {
		rspamd_fuzzy_backend_count(ctx->backend, fuzzy_count_callback, ctx);

		msg_info("successfully updated fuzzy storage %s: %d updates in queue; "
				 "%d pending currently; "
				 "%d added; %d deleted; %d extended; %d duplicates",
				 ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
				 cbdata->updates_pending->len,
				 ctx->updates_pending->len,
				 nadded, ndeleted, nextended, nignored);
		rspamd_fuzzy_backend_version(ctx->backend, source,
									 fuzzy_update_version_callback, NULL);
		ctx->updates_failed = 0;

		if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) {
			/* Plan exit */
			ev_break(ctx->event_loop, EVBREAK_ALL);
		}
	}
	else {
		if (++ctx->updates_failed > ctx->updates_maxfail) {
			msg_err("cannot commit update transaction to fuzzy backend %s, discard "
					"%ud updates after %d retries",
					ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
					cbdata->updates_pending->len,
					ctx->updates_maxfail);
			ctx->updates_failed = 0;

			if (cbdata->final || ctx->worker->state != rspamd_worker_state_running) {
				/* Plan exit */
				ev_break(ctx->event_loop, EVBREAK_ALL);
			}
		}
		else {
			if (ctx->updates_pending) {
				msg_err("cannot commit update transaction to fuzzy backend %s; "
						"%ud updates are still left; %ud currently pending;"
						" %d retries remaining",
						ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
						cbdata->updates_pending->len,
						ctx->updates_pending->len,
						ctx->updates_maxfail - ctx->updates_failed);
				/* Move the remaining updates to ctx queue */
				g_array_append_vals(ctx->updates_pending,
									cbdata->updates_pending->data,
									cbdata->updates_pending->len);

				if (cbdata->final) {
					/* Try one more time */
					rspamd_fuzzy_process_updates_queue(cbdata->ctx, cbdata->source,
													   cbdata->final);
				}
			}
			else {
				/* We have lost our ctx, so it is a race condition case */
				msg_err("cannot commit update transaction to fuzzy backend %s; "
						"%ud updates are still left; no more retries: a worker is terminated",
						ctx->worker->cf->bind_conf ? ctx->worker->cf->bind_conf->bind_line : "unknown",
						cbdata->updates_pending->len);
			}
		}
	}

	g_array_free(cbdata->updates_pending, TRUE);
	g_free(cbdata->source);
	g_free(cbdata);
}

static gboolean
rspamd_fuzzy_process_updates_queue(struct rspamd_fuzzy_storage_ctx *ctx,
								   const char *source, gboolean final)
{

	struct rspamd_updates_cbdata *cbdata;

	if (ctx->updates_pending->len > 0) {
		cbdata = g_malloc(sizeof(*cbdata));
		cbdata->ctx = ctx;
		cbdata->final = final;
		cbdata->updates_pending = ctx->updates_pending;
		ctx->updates_pending = g_array_sized_new(FALSE, FALSE,
												 sizeof(struct fuzzy_peer_cmd),
												 MAX(cbdata->updates_pending->len, 1024));
		cbdata->source = g_strdup(source);
		rspamd_fuzzy_backend_process_updates(ctx->backend,
											 cbdata->updates_pending,
											 source, rspamd_fuzzy_updates_cb, cbdata);
		return TRUE;
	}
	else if (final) {
		/* No need to sync */
		ev_break(ctx->event_loop, EVBREAK_ALL);
	}

	return FALSE;
}

static void
rspamd_fuzzy_reply_io(EV_P_ ev_io *w, int revents)
{
	struct fuzzy_session *session = (struct fuzzy_session *) w->data;

	ev_io_stop(EV_A_ w);
	rspamd_fuzzy_write_reply(session);
	REF_RELEASE(session);
}

static void
rspamd_fuzzy_tcp_enqueue_reply(struct fuzzy_session *session)
{
	struct fuzzy_tcp_session *tcp_session = session->tcp_session;
	struct fuzzy_tcp_reply_queue_elt *reply_elt;
	gsize len;
	gconstpointer data;

	if (tcp_session == NULL) {
		msg_err("internal error: tcp_session is NULL in rspamd_fuzzy_tcp_enqueue_reply");
		return;
	}

	if (tcp_session->closed) {
		/*
		 * A command that outlived its connection (e.g. a Redis reply arriving
		 * after EOF or timeout). There is nowhere to write it: the socket is
		 * gone, and restarting the watcher here would resurrect I/O on a dead
		 * session.
		 */
		msg_debug_fuzzy_storage("discard reply for the closed TCP session from %s",
								rspamd_inet_address_to_string(session->addr));

		return;
	}

	/* Determine reply data and length */
	if (session->cmd_type == CMD_ENCRYPTED_NORMAL ||
		session->cmd_type == CMD_ENCRYPTED_SHINGLE) {
		/* Encrypted reply */
		if (session->epoch >= RSPAMD_FUZZY_EPOCH12) {
			data = &session->reply.v2;
			len = sizeof(session->reply.v2);
		}
		else if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
			data = &session->reply.v1;
			len = sizeof(session->reply.v1);
		}
		else {
			data = &session->reply.v1;
			len = sizeof(session->reply.v1.hdr) + sizeof(session->reply.v1.rep.v1);
		}
	}
	else {
		if (session->epoch >= RSPAMD_FUZZY_EPOCH12) {
			data = &session->reply.v2.rep;
			len = sizeof(session->reply.v2.rep);
		}
		else if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
			data = &session->reply.v1.rep;
			len = sizeof(session->reply.v1.rep);
		}
		else {
			data = &session->reply.v1.rep;
			len = sizeof(session->reply.v1.rep.v1);
		}
	}

	/* Create reply queue element */
	reply_elt = g_malloc0(sizeof(*reply_elt));
	reply_elt->rep.size_hdr = GUINT16_TO_LE((uint16_t) len);
	memcpy(&reply_elt->rep.payload, data, len);
	reply_elt->written = 0;

	/* Add to queue */
	DL_APPEND(tcp_session->replies_queue, reply_elt);

	msg_debug_fuzzy_storage("enqueued TCP reply to %s, %z bytes",
							rspamd_inet_address_to_string(session->addr),
							len);

	/* Enable write event if not already enabled */
	if (ev_is_active(&tcp_session->common.io)) {
		int events = tcp_session->common.io.events;
		if (!(events & EV_WRITE)) {
			ev_io_stop(tcp_session->common.ctx->event_loop, &tcp_session->common.io);
			ev_io_set(&tcp_session->common.io, tcp_session->common.fd, EV_READ | EV_WRITE);
			ev_io_start(tcp_session->common.ctx->event_loop, &tcp_session->common.io);
		}
	}
	else {
		/* Watcher is not active, start it with both read and write */
		ev_io_set(&tcp_session->common.io, tcp_session->common.fd, EV_READ | EV_WRITE);
		ev_io_start(tcp_session->common.ctx->event_loop, &tcp_session->common.io);
	}
}

static void
rspamd_fuzzy_write_reply(struct fuzzy_session *session)
{
	gssize r;
	gsize len;
	gconstpointer data;

	/* Check if this is a TCP session */
	if (session->tcp_session != NULL) {
		rspamd_fuzzy_tcp_enqueue_reply(session);
		return;
	}

	if (session->cmd_type == CMD_ENCRYPTED_NORMAL ||
		session->cmd_type == CMD_ENCRYPTED_SHINGLE) {
		/* Encrypted reply */
		if (session->epoch >= RSPAMD_FUZZY_EPOCH12) {
			data = &session->reply.v2;
			len = sizeof(session->reply.v2);
		}
		else if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
			data = &session->reply.v1;
			len = sizeof(session->reply.v1);
		}
		else {
			data = &session->reply.v1;
			len = sizeof(session->reply.v1.hdr) + sizeof(session->reply.v1.rep.v1);
		}
	}
	else {
		if (session->epoch >= RSPAMD_FUZZY_EPOCH12) {
			data = &session->reply.v2.rep;
			len = sizeof(session->reply.v2.rep);
		}
		else if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
			data = &session->reply.v1.rep;
			len = sizeof(session->reply.v1.rep);
		}
		else {
			data = &session->reply.v1.rep;
			len = sizeof(session->reply.v1.rep.v1);
		}
	}

	r = rspamd_inet_address_sendto(session->fd, data, len, 0,
								   session->addr);

	if (r == -1) {
		if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) {
			/* Grab reference to avoid early destruction */
			REF_RETAIN(session);
			session->io.data = session;
			ev_io_init(&session->io,
					   rspamd_fuzzy_reply_io, session->fd, EV_WRITE);
			ev_io_start(session->ctx->event_loop, &session->io);
		}
		else {
			msg_err("error while writing reply: %s", strerror(errno));
		}
	}
}

static void
rspamd_fuzzy_update_key_stat(gboolean matched,
							 struct fuzzy_key_stat *key_stat,
							 unsigned int cmd,
							 struct rspamd_fuzzy_reply *res,
							 ev_tstamp timestamp)
{
	if (!matched && res->v1.value != 0) {
		key_stat->errors++;
	}
	else {
		if (cmd == FUZZY_CHECK) {
			key_stat->checked++;

			if (matched) {
				key_stat->matched++;
			}

			if (G_UNLIKELY(key_stat->last_checked_time == 0.0)) {
				key_stat->last_checked_time = timestamp;
				key_stat->last_checked_count = key_stat->checked;
				key_stat->last_matched_count = key_stat->matched;
			}
			else if (G_UNLIKELY(timestamp > key_stat->last_checked_time + KEY_STAT_INTERVAL)) {
				uint64_t nchecked = key_stat->checked - key_stat->last_checked_count;
				uint64_t nmatched = key_stat->matched - key_stat->last_matched_count;

				rspamd_set_counter_ema(&key_stat->checked_ctr, nchecked, 0.5f);
				rspamd_set_counter_ema(&key_stat->matched_ctr, nmatched, 0.5f);
				key_stat->last_checked_time = timestamp;
				key_stat->last_checked_count = key_stat->checked;
				key_stat->last_matched_count = key_stat->matched;
			}
		}
		else if (cmd == FUZZY_WRITE) {
			key_stat->added++;
		}
		else if (cmd == FUZZY_DEL) {
			key_stat->deleted++;
		}
	}
}

static void
rspamd_fuzzy_update_stats(struct rspamd_fuzzy_storage_ctx *ctx,
						  enum rspamd_fuzzy_epoch epoch,
						  gboolean matched,
						  gboolean is_shingle,
						  gboolean is_delayed,
						  struct fuzzy_key *key,
						  struct fuzzy_key_stat *ip_stat,
						  unsigned int cmd,
						  struct rspamd_fuzzy_reply *res,
						  ev_tstamp timestamp)
{
	ctx->stat.fuzzy_hashes_checked[epoch]++;

	if (matched) {
		ctx->stat.fuzzy_hashes_found[epoch]++;
	}
	if (is_shingle) {
		ctx->stat.fuzzy_shingles_checked[epoch]++;
	}
	if (is_delayed) {
		ctx->stat.delayed_hashes++;
	}

	if (key == NULL && ip_stat != NULL && ctx->unkeyed_stat != NULL) {
		/* Unkeyed client: update the aggregate bucket */
		rspamd_fuzzy_update_key_stat(matched, ctx->unkeyed_stat, cmd, res, timestamp);
	}

	if (key) {
		rspamd_fuzzy_update_key_stat(matched, key->stat, cmd, res, timestamp);

		if (matched || ((cmd == FUZZY_WRITE || cmd == FUZZY_DEL) && res->v1.value == 0)) {
			/* Update per flag stats */
			khiter_t k;
			struct fuzzy_key_stat *flag_stat;
			k = kh_get(fuzzy_key_flag_stat, key->flags_stat, res->v1.flag);

			if (k == kh_end(key->flags_stat)) {
				/* Insert new flag */
				int r;
				k = kh_put(fuzzy_key_flag_stat, key->flags_stat, res->v1.flag, &r);
				memset(&kh_value(key->flags_stat, k), 0, sizeof(struct fuzzy_key_stat));
			}

			flag_stat = &kh_value(key->flags_stat, k);
			rspamd_fuzzy_update_key_stat(matched, flag_stat, cmd, res, timestamp);
		}
	}

	if (ip_stat) {
		if (!matched && res->v1.value != 0) {
			ip_stat->errors++;
		}
		else {
			if (cmd == FUZZY_CHECK) {
				ip_stat->checked++;

				if (matched) {
					ip_stat->matched++;
				}
			}
			else if (cmd == FUZZY_WRITE) {
				ip_stat->added++;
			}
			else if (cmd == FUZZY_DEL) {
				ip_stat->deleted++;
			}
		}
	}
}

enum rspamd_fuzzy_reply_flags {
	RSPAMD_FUZZY_REPLY_ENCRYPTED = 0x1u << 0u,
	RSPAMD_FUZZY_REPLY_SHINGLE = 0x1u << 1u,
	RSPAMD_FUZZY_REPLY_DELAY = 0x1u << 2u,
	/* The reply digest is the actual digest resolved by the backend */
	RSPAMD_FUZZY_REPLY_MATCHED = 0x1u << 3u,
};

/*
 * Check if a flag ID is forbidden by the given set or per-key set.
 * Returns true if the flag should be suppressed.
 */
static bool
rspamd_fuzzy_flag_is_forbidden(struct fuzzy_session *session,
							   uint32_t flag, int reply_flags)
{
	khiter_t k;

	/* A key with its own forbidden_ids set REPLACES the default:
	 * an empty per-key set means "nothing forbidden for this key";
	 * default_forbidden_ids only covers keys with no ACL of their own.
	 * Mirrors the v1 reply path (default_disabled fallback there). */
	if ((reply_flags & RSPAMD_FUZZY_REPLY_ENCRYPTED) &&
		session->key && session->key->forbidden_ids) {
		k = kh_get(fuzzy_key_ids_set, session->key->forbidden_ids, flag);
		return k != kh_end(session->key->forbidden_ids);
	}

	k = kh_get(fuzzy_key_ids_set, session->ctx->default_forbidden_ids, flag);
	return k != kh_end(session->ctx->default_forbidden_ids);
}

/*
 * Filter forbidden flags from a v2 reply.
 * Removes forbidden extra flags in-place. If the primary flag is forbidden,
 * promotes the first valid extra flag to primary; if all flags are forbidden,
 * zeroes the entire reply.
 */
static void
rspamd_fuzzy_filter_forbidden_v2(struct rspamd_fuzzy_reply_v2 *rep_v2,
								 struct fuzzy_session *session,
								 int reply_flags)
{
	/* First, filter forbidden extra flags */
	uint8_t dst = 0;
	for (uint8_t i = 0; i < rep_v2->n_extra_flags; i++) {
		if (!rspamd_fuzzy_flag_is_forbidden(session,
											rep_v2->extra_flags[i].flag, reply_flags)) {
			if (dst != i) {
				rep_v2->extra_flags[dst] = rep_v2->extra_flags[i];
			}
			dst++;
		}
	}
	rep_v2->n_extra_flags = dst;

	/* Check if primary flag is forbidden */
	if (rspamd_fuzzy_flag_is_forbidden(session, rep_v2->v1.flag, reply_flags)) {
		if (rep_v2->n_extra_flags > 0) {
			/* Promote the first valid extra flag to primary */
			rep_v2->v1.value = rep_v2->extra_flags[0].value;
			rep_v2->v1.flag = rep_v2->extra_flags[0].flag;
			/* Shift remaining extras down */
			for (uint8_t i = 1; i < rep_v2->n_extra_flags; i++) {
				rep_v2->extra_flags[i - 1] = rep_v2->extra_flags[i];
			}
			rep_v2->n_extra_flags--;
		}
		else {
			/* All flags forbidden — suppress the entire result */
			rep_v2->ts = 0;
			rep_v2->v1.prob = 0.0f;
			rep_v2->v1.value = 0;
			rep_v2->v1.flag = 0;
		}
	}
}

static void
rspamd_fuzzy_make_reply(struct rspamd_fuzzy_cmd *cmd,
						struct rspamd_fuzzy_reply *result,
						struct rspamd_fuzzy_multiflag_result *mf_result,
						struct fuzzy_session *session,
						int flags)
{
	gsize len;

	if (cmd) {
		result->v1.tag = cmd->tag;

		if (session->epoch >= RSPAMD_FUZZY_EPOCH12) {
			/* Populate v2 reply with multi-flag data */
			struct rspamd_fuzzy_reply_v2 *rep_v2 = &session->reply.v2.rep;
			memset(rep_v2, 0, sizeof(*rep_v2));

			/* Primary flag goes into v1 sub-struct */
			rep_v2->v1 = result->v1;
			memcpy(rep_v2->digest, result->digest, sizeof(rep_v2->digest));
			rep_v2->ts = result->ts;

			/* Extra flags from multiflag result */
			if (mf_result) {
				uint8_t n_extra = mf_result->n_extra_flags;
				/*
				 * rep_v2->extra_flags is fixed-size [RSPAMD_FUZZY_MAX_EXTRA_FLAGS].
				 * All current backends bound n_extra_flags to this maximum, but
				 * clamp defensively here so a future backend bug cannot turn
				 * into an OOB write on the reply struct.
				 */
				if (G_UNLIKELY(n_extra > RSPAMD_FUZZY_MAX_EXTRA_FLAGS)) {
					msg_warn("backend returned n_extra_flags=%d > max %d, clamping",
							 (int) n_extra, (int) RSPAMD_FUZZY_MAX_EXTRA_FLAGS);
					n_extra = RSPAMD_FUZZY_MAX_EXTRA_FLAGS;
				}
				rep_v2->n_extra_flags = n_extra;
				memcpy(rep_v2->extra_flags, mf_result->extra_flags,
					   sizeof(struct rspamd_fuzzy_flag_entry) * n_extra);
			}

			if (flags & RSPAMD_FUZZY_REPLY_DELAY) {
				rep_v2->ts = 0;
				rep_v2->v1.prob = 0.0f;
				rep_v2->v1.value = 0;
			}

			/* Filter forbidden flags from primary and extra flags */
			rspamd_fuzzy_filter_forbidden_v2(rep_v2, session, flags);

			if ((flags & RSPAMD_FUZZY_REPLY_MATCHED) && rep_v2->v1.prob > 0.5f) {
				rep_v2->reserved[0] |= RSPAMD_FUZZY_REPLY_FLAG_MATCHED_DIGEST;
			}

			if (flags & RSPAMD_FUZZY_REPLY_ENCRYPTED) {

				/* Use a temporary v1 reply for stats (stats API expects rspamd_fuzzy_reply) */
				struct rspamd_fuzzy_reply stats_rep;
				memset(&stats_rep, 0, sizeof(stats_rep));
				stats_rep.v1 = rep_v2->v1;
				memcpy(stats_rep.digest, rep_v2->digest, sizeof(stats_rep.digest));
				stats_rep.ts = rep_v2->ts;

				ottery_rand_bytes(session->reply.v2.hdr.nonce,
								  sizeof(session->reply.v2.hdr.nonce));

				len = sizeof(session->reply.v2.rep);

				if (cmd->cmd != FUZZY_STAT && cmd->cmd <= FUZZY_CLIENT_MAX) {
					rspamd_fuzzy_update_stats(session->ctx,
											  session->epoch,
											  rep_v2->v1.prob > 0.5f,
											  flags & RSPAMD_FUZZY_REPLY_SHINGLE,
											  flags & RSPAMD_FUZZY_REPLY_DELAY,
											  session->key,
											  session->ip_stat,
											  cmd->cmd,
											  &stats_rep,
											  session->timestamp);
				}

				rspamd_cryptobox_encrypt_nm_inplace((unsigned char *) &session->reply.v2.rep,
													len,
													session->reply.v2.hdr.nonce,
													session->nm,
													session->reply.v2.hdr.mac);
			}
			else {
				struct rspamd_fuzzy_reply stats_rep;
				memset(&stats_rep, 0, sizeof(stats_rep));
				stats_rep.v1 = rep_v2->v1;
				memcpy(stats_rep.digest, rep_v2->digest, sizeof(stats_rep.digest));
				stats_rep.ts = rep_v2->ts;

				if (cmd->cmd != FUZZY_STAT && cmd->cmd <= FUZZY_CLIENT_MAX) {
					rspamd_fuzzy_update_stats(session->ctx,
											  session->epoch,
											  rep_v2->v1.prob > 0.5f,
											  flags & RSPAMD_FUZZY_REPLY_SHINGLE,
											  flags & RSPAMD_FUZZY_REPLY_DELAY,
											  session->key,
											  session->ip_stat,
											  cmd->cmd,
											  &stats_rep,
											  session->timestamp);
				}
			}
		}
		else {
			/* EPOCH10/EPOCH11 path — v1 reply */
			memcpy(&session->reply.v1.rep, result, sizeof(*result));

			if (flags & RSPAMD_FUZZY_REPLY_DELAY) {
				session->reply.v1.rep.ts = 0;
				session->reply.v1.rep.v1.prob = 0.0f;
				session->reply.v1.rep.v1.value = 0;
			}

			if ((flags & RSPAMD_FUZZY_REPLY_MATCHED) &&
				session->reply.v1.rep.v1.prob > 0.5f) {
				session->reply.v1.rep.reserved[0] |= RSPAMD_FUZZY_REPLY_FLAG_MATCHED_DIGEST;
			}

			bool default_disabled = false;
			{
				khiter_t k;
				k = kh_get(fuzzy_key_ids_set, session->ctx->default_forbidden_ids, session->reply.v1.rep.v1.flag);
				if (k != kh_end(session->ctx->default_forbidden_ids)) {
					default_disabled = true;
				}
			}

			if (flags & RSPAMD_FUZZY_REPLY_ENCRYPTED) {
				/* No prob check here: a key with its own forbidden_ids
				 * overrides the default regardless of the match result,
				 * otherwise a delayed reply (prob is zeroed, flag is not)
				 * would fall back to default_forbidden_ids. */
				if (session->key && session->key->forbidden_ids) {
					khiter_t k;
					k = kh_get(fuzzy_key_ids_set, session->key->forbidden_ids, session->reply.v1.rep.v1.flag);
					if (k != kh_end(session->key->forbidden_ids)) {
						session->reply.v1.rep.ts = 0;
						session->reply.v1.rep.v1.prob = 0.0f;
						session->reply.v1.rep.v1.value = 0;
						session->reply.v1.rep.v1.flag = 0;
					}
				}
				else if (default_disabled) {
					session->reply.v1.rep.ts = 0;
					session->reply.v1.rep.v1.prob = 0.0f;
					session->reply.v1.rep.v1.value = 0;
					session->reply.v1.rep.v1.flag = 0;
				}

				ottery_rand_bytes(session->reply.v1.hdr.nonce,
								  sizeof(session->reply.v1.hdr.nonce));

				if (session->epoch > RSPAMD_FUZZY_EPOCH10) {
					len = sizeof(session->reply.v1.rep);
				}
				else {
					len = sizeof(session->reply.v1.rep.v1);
				}

				if (cmd->cmd != FUZZY_STAT && cmd->cmd <= FUZZY_CLIENT_MAX) {
					rspamd_fuzzy_update_stats(session->ctx,
											  session->epoch,
											  session->reply.v1.rep.v1.prob > 0.5f,
											  flags & RSPAMD_FUZZY_REPLY_SHINGLE,
											  flags & RSPAMD_FUZZY_REPLY_DELAY,
											  session->key,
											  session->ip_stat,
											  cmd->cmd,
											  &session->reply.v1.rep,
											  session->timestamp);
				}

				rspamd_cryptobox_encrypt_nm_inplace((unsigned char *) &session->reply.v1.rep,
													len,
													session->reply.v1.hdr.nonce,
													session->nm,
													session->reply.v1.hdr.mac);
			}
			else if (default_disabled) {
				session->reply.v1.rep.ts = 0;
				session->reply.v1.rep.v1.prob = 0.0f;
				session->reply.v1.rep.v1.value = 0;
				session->reply.v1.rep.v1.flag = 0;
			}
			if (!(flags & RSPAMD_FUZZY_REPLY_ENCRYPTED)) {
				if (cmd->cmd != FUZZY_STAT && cmd->cmd <= FUZZY_CLIENT_MAX) {
					rspamd_fuzzy_update_stats(session->ctx,
											  session->epoch,
											  session->reply.v1.rep.v1.prob > 0.5f,
											  flags & RSPAMD_FUZZY_REPLY_SHINGLE,
											  flags & RSPAMD_FUZZY_REPLY_DELAY,
											  session->key,
											  session->ip_stat,
											  cmd->cmd,
											  &session->reply.v1.rep,
											  session->timestamp);
				}
			}
		}
	}

	rspamd_fuzzy_write_reply(session);
}

static enum fuzzy_peer_send_status
fuzzy_peer_try_send(int fd, struct fuzzy_peer_request *up_req)
{
	const unsigned char *p = (const unsigned char *) &up_req->cmd + up_req->written;
	gsize remaining = sizeof(up_req->cmd) - up_req->written;
	gssize r;

	r = write(fd, p, remaining);

	if (r > 0) {
		up_req->written += (gsize) r;
		if (up_req->written == sizeof(up_req->cmd)) {
			return FUZZY_PEER_SEND_DONE;
		}
		/* Short write — kernel buffer probably filled; resume from up_req->written. */
		return FUZZY_PEER_SEND_AGAIN;
	}

	if (r == 0) {
		/* write(2) only returns 0 if asked to write 0 bytes; not expected here */
		return FUZZY_PEER_SEND_AGAIN;
	}

	if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
		return FUZZY_PEER_SEND_AGAIN;
	}

	return FUZZY_PEER_SEND_FATAL;
}

static void
fuzzy_peer_request_release(struct fuzzy_peer_request *up_req)
{
	if (up_req->ctx != NULL) {
		DL_DELETE(up_req->ctx->pending_peer_requests, up_req);
	}
	g_free(up_req);
}

static void
fuzzy_peer_send_io(EV_P_ ev_io *w, int revents)
{
	struct fuzzy_peer_request *up_req = (struct fuzzy_peer_request *) w->data;
	enum fuzzy_peer_send_status status = fuzzy_peer_try_send(w->fd, up_req);

	if (status == FUZZY_PEER_SEND_AGAIN) {
		/* Leave the watcher running; libev will fire again when the pipe is writable. */
		return;
	}

	if (status == FUZZY_PEER_SEND_FATAL) {
		msg_err("cannot send update request to the peer: %s", strerror(errno));
	}

	ev_io_stop(EV_A_ w);
	fuzzy_peer_request_release(up_req);
}

/*
 * Decodes the sender facts bit field and sets the `sender` field of the
 * extensions table that must be on the top of the stack.
 *
 * Values are pushed as strings: a handler should not need to reimplement this
 * table and the raw numbers mean nothing in a log line. Fields that the sender
 * facts describe as absent or unknown are not set at all.
 */
static void
rspamd_fuzzy_sender_facts_tolua(lua_State *L,
								struct rspamd_fuzzy_cmd_extension *ext)
{
	uint32_t val;
	unsigned int cls;
	const char *str;

	/* The parser rejects any other length, this is merely belt and braces */
	if (ext->length < RSPAMD_FUZZY_SENDER_FACTS_LEN) {
		return;
	}

	memcpy(&val, ext->payload, sizeof(val));
	val = GUINT32_FROM_BE(val);
	cls = (val >> RSPAMD_FUZZY_SENDER_FACTS_CLASS_SHIFT) &
		  RSPAMD_FUZZY_SENDER_FACTS_CLASS_MASK;

	if (cls != RSPAMD_FUZZY_SENDER_FACTS_CLASS_V0) {
		/* Some future layout we know nothing about */
		return;
	}

	lua_createtable(L, 0, 7);

	if ((str = rspamd_fuzzy_sf_spf_str((val >> RSPAMD_FUZZY_SF_SPF_SHIFT) &
									   RSPAMD_FUZZY_SF_SPF_MASK)) != NULL) {
		lua_pushstring(L, str);
		lua_setfield(L, -2, "spf");
	}

	if ((str = rspamd_fuzzy_sf_dkim_str((val >> RSPAMD_FUZZY_SF_DKIM_SHIFT) &
										RSPAMD_FUZZY_SF_DKIM_MASK)) != NULL) {
		lua_pushstring(L, str);
		lua_setfield(L, -2, "dkim");
	}

	if ((str = rspamd_fuzzy_sf_dmarc_str((val >> RSPAMD_FUZZY_SF_DMARC_SHIFT) &
										 RSPAMD_FUZZY_SF_DMARC_MASK)) != NULL) {
		lua_pushstring(L, str);
		lua_setfield(L, -2, "dmarc");
	}

	if ((str = rspamd_fuzzy_sf_ptr_str((val >> RSPAMD_FUZZY_SF_PTR_SHIFT) &
									   RSPAMD_FUZZY_SF_PTR_MASK)) != NULL) {
		lua_pushstring(L, str);
		lua_setfield(L, -2, "ptr");
	}

	lua_pushboolean(L, (val >> RSPAMD_FUZZY_SF_PTR_GENERIC_SHIFT) & 1u);
	lua_setfield(L, -2, "ptr_generic");

	lua_pushstring(L, rspamd_fuzzy_sf_rcpts_str((val >> RSPAMD_FUZZY_SF_RCPTS_SHIFT) &
												RSPAMD_FUZZY_SF_RCPTS_MASK));
	lua_setfield(L, -2, "rcpts");

	lua_pushboolean(L, (val >> RSPAMD_FUZZY_SF_TLS_SHIFT) & 1u);
	lua_setfield(L, -2, "tls");

	lua_setfield(L, -2, "sender");
}

static void
rspamd_fuzzy_extensions_tolua(lua_State *L,
							  struct fuzzy_session *session)
{
	struct rspamd_fuzzy_cmd_extension *ext;
	rspamd_inet_addr_t *addr;

	lua_createtable(L, 0, 0);

	LL_FOREACH(session->extensions, ext)
	{
		switch (ext->ext) {
		case RSPAMD_FUZZY_EXT_SOURCE_DOMAIN:
			lua_pushlstring(L, ext->payload, ext->length);
			lua_setfield(L, -2, "domain");
			break;
		case RSPAMD_FUZZY_EXT_SOURCE_IP4:
			addr = rspamd_inet_address_new(AF_INET, ext->payload);
			rspamd_lua_ip_push(L, addr);
			rspamd_inet_address_free(addr);
			lua_setfield(L, -2, "ip");
			break;
		case RSPAMD_FUZZY_EXT_SOURCE_IP6:
			addr = rspamd_inet_address_new(AF_INET6, ext->payload);
			rspamd_lua_ip_push(L, addr);
			rspamd_inet_address_free(addr);
			lua_setfield(L, -2, "ip");
			break;
		case RSPAMD_FUZZY_EXT_SENDER_FACTS:
			rspamd_fuzzy_sender_facts_tolua(L, ext);
			break;
		}
	}
}

static void
rspamd_fuzzy_check_callback(struct rspamd_fuzzy_multiflag_result *mf_result, void *ud)
{
	struct fuzzy_session *session = ud;
	gboolean is_shingle = FALSE, __attribute__((unused)) encrypted = FALSE;
	struct rspamd_fuzzy_cmd *cmd = NULL;
	const struct rspamd_shingle *shingle = NULL;
	struct rspamd_shingle sgl_cpy;
	struct rspamd_fuzzy_reply *result = &mf_result->rep;
	int send_flags = 0;

	switch (session->cmd_type) {
	case CMD_ENCRYPTED_NORMAL:
		encrypted = TRUE;
		send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
		/* Fallthrough */
	case CMD_NORMAL:
		cmd = &session->cmd.basic;
		break;

	case CMD_ENCRYPTED_SHINGLE:
		encrypted = TRUE;
		send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
		/* Fallthrough */
	case CMD_SHINGLE:
		cmd = &session->cmd.basic;
		memcpy(&sgl_cpy, &session->cmd.sgl, sizeof(sgl_cpy));
		shingle = &sgl_cpy;
		is_shingle = TRUE;
		send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE;
		break;
	}

	if (session->ctx->lua_post_handlers != NULL) {
		struct rspamd_lua_fuzzy_script *cur;
		LL_FOREACH(session->ctx->lua_post_handlers, cur)
		{
			/* Start lua post handler */
			lua_State *L = session->ctx->cfg->lua_state;
			int err_idx, ret, nargs = 10;

			lua_pushcfunction(L, &rspamd_lua_traceback);
			err_idx = lua_gettop(L);
			/* Preallocate stack (small opt) */
			lua_checkstack(L, err_idx + nargs + 9);
			/* function */
			lua_rawgeti(L, LUA_REGISTRYINDEX, cur->cbref);
			/* client IP */
			if (session->addr) {
				rspamd_lua_ip_push(L, session->addr);
			}
			else {
				lua_pushnil(L);
			}
			/* client command */
			lua_pushinteger(L, cmd->cmd);
			/* command value (push as rspamd_text) */
			(void) lua_new_text(L, result->digest, sizeof(result->digest), FALSE);
			/* is shingle */
			lua_pushboolean(L, is_shingle);
			/* result value */
			lua_pushinteger(L, result->v1.value);
			/* result probability */
			lua_pushnumber(L, result->v1.prob);
			/* result flag */
			lua_pushinteger(L, result->v1.flag);
			/* result timestamp */
			lua_pushinteger(L, result->ts);
			/* TODO: add additional data maybe (encryption, pubkey, etc) */
			rspamd_fuzzy_extensions_tolua(L, session);
			/* We push shingles merely for commands that modify content to avoid extra work */
			if (is_shingle && cmd->cmd != FUZZY_CHECK) {
				lua_newshingle(L, &session->cmd.sgl);
			}
			else {
				lua_pushnil(L);
			}

			if ((ret = lua_pcall(L, nargs, LUA_MULTRET, err_idx)) != 0) {
				msg_err("call to lua_post_handler lua "
						"script failed (%d): %s",
						ret, lua_tostring(L, -1));
			}
			else {
				/* Return values order:
				 * the first reply will be on err_idx + 1
				 * if it is true, then we need to read the former ones:
				 * 2-nd will be reply code
				 * 3-rd will be probability (or 0.0 if missing)
				 * 4-th value is flag (or default flag if missing)
				 */
				ret = lua_toboolean(L, err_idx + 1);

				if (ret) {
					/* Artificial reply */
					result->v1.value = lua_tointeger(L, err_idx + 2);

					if (lua_isnumber(L, err_idx + 3)) {
						result->v1.prob = lua_tonumber(L, err_idx + 3);
					}
					else {
						result->v1.prob = 0.0f;
					}

					if (lua_isnumber(L, err_idx + 4)) {
						result->v1.flag = lua_tointeger(L, err_idx + 4);
					}

					lua_settop(L, 0);
					rspamd_fuzzy_make_reply(cmd, result, mf_result, session, send_flags);
					REF_RELEASE(session);

					return;
				}
			}

			lua_settop(L, 0);
		}
	}

	/* Check if the returned hash from fuzzy matching should be skipped */
	if (session->ctx->skip_hashes && result->v1.value > 0) {
		char hexbuf[sizeof(result->digest) * 2 + 1];
		rspamd_encode_hex_buf(result->digest, sizeof(result->digest),
							  hexbuf, sizeof(hexbuf) - 1);
		hexbuf[sizeof(hexbuf) - 1] = '\0';

		if (rspamd_match_hash_map(session->ctx->skip_hashes,
								  hexbuf, sizeof(hexbuf) - 1)) {
			result->v1.value = 401;
			result->v1.prob = 0.0f;
		}
	}

	if (!isnan(session->ctx->delay) &&
		rspamd_match_radix_map_addr(session->ctx->delay_whitelist,
									session->addr) == NULL) {
		double hash_age = rspamd_get_calendar_ticks() - result->ts;
		double jittered_age = rspamd_time_jitter(session->ctx->delay,
												 session->ctx->delay / 2.0);

		if (hash_age < jittered_age) {
			send_flags |= RSPAMD_FUZZY_REPLY_DELAY;
		}
	}

	/* Refresh hash if found with strong confidence */
	if (result->v1.prob > 0.9 && !session->ctx->read_only) {
		struct fuzzy_peer_cmd up_cmd;
		struct fuzzy_peer_request *up_req;

		if (session->worker->index == 0) {
			/* Just add to the queue */
			memset(&up_cmd, 0, sizeof(up_cmd));
			up_cmd.is_shingle = is_shingle;
			memcpy(up_cmd.cmd.normal.digest, result->digest,
				   sizeof(up_cmd.cmd.normal.digest));
			up_cmd.cmd.normal.flag = result->v1.flag;
			up_cmd.cmd.normal.cmd = FUZZY_REFRESH;
			up_cmd.cmd.normal.shingles_count = cmd->shingles_count;

			if (is_shingle && shingle) {
				memcpy(&up_cmd.cmd.shingle.sgl, shingle,
					   sizeof(up_cmd.cmd.shingle.sgl));
			}

			g_array_append_val(session->ctx->updates_pending, up_cmd);
		}
		else {
			/* We need to send request to the peer */
			up_req = g_malloc0(sizeof(*up_req));
			up_req->cmd.is_shingle = is_shingle;

			memcpy(up_req->cmd.cmd.normal.digest, result->digest,
				   sizeof(up_req->cmd.cmd.normal.digest));
			up_req->cmd.cmd.normal.flag = result->v1.flag;
			up_req->cmd.cmd.normal.cmd = FUZZY_REFRESH;
			up_req->cmd.cmd.normal.shingles_count = cmd->shingles_count;

			if (is_shingle && shingle) {
				memcpy(&up_req->cmd.cmd.shingle.sgl, shingle,
					   sizeof(up_req->cmd.cmd.shingle.sgl));
			}

			enum fuzzy_peer_send_status sst = fuzzy_peer_try_send(
				session->ctx->peer_fd, up_req);

			if (sst == FUZZY_PEER_SEND_DONE) {
				g_free(up_req);
			}
			else if (sst == FUZZY_PEER_SEND_AGAIN) {
				up_req->ctx = session->ctx;
				DL_APPEND(session->ctx->pending_peer_requests, up_req);
				up_req->io_ev.data = up_req;
				ev_io_init(&up_req->io_ev, fuzzy_peer_send_io,
						   session->ctx->peer_fd, EV_WRITE);
				ev_io_start(session->ctx->event_loop, &up_req->io_ev);
			}
			else {
				msg_err("cannot send update request to the peer: %s",
						strerror(errno));
				g_free(up_req);
			}
		}
	}

	if (result->v1.prob > 0.5) {
		/* The digest in the reply is the one resolved by the backend */
		send_flags |= RSPAMD_FUZZY_REPLY_MATCHED;
	}

	rspamd_fuzzy_make_reply(cmd, result, mf_result, session, send_flags);

	REF_RELEASE(session);
}

static void
rspamd_fuzzy_process_command(struct fuzzy_session *session)
{
	gboolean is_shingle = FALSE, __attribute__((unused)) encrypted = FALSE;
	struct rspamd_fuzzy_cmd *cmd = NULL;
	struct rspamd_fuzzy_reply result;
	struct fuzzy_peer_cmd up_cmd;
	struct fuzzy_peer_request *up_req;
	struct fuzzy_key_stat *ip_stat = NULL;
	char hexbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
	rspamd_inet_addr_t *naddr;
	gpointer ptr;
	gsize up_len = 0;
	int send_flags = 0;

	cmd = &session->cmd.basic;

	switch (session->cmd_type) {
	case CMD_NORMAL:
		up_len = sizeof(session->cmd.basic);
		break;
	case CMD_SHINGLE:
		up_len = sizeof(session->cmd);
		is_shingle = TRUE;
		send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE;
		break;
	case CMD_ENCRYPTED_NORMAL:
		up_len = sizeof(session->cmd.basic);
		encrypted = TRUE;
		send_flags |= RSPAMD_FUZZY_REPLY_ENCRYPTED;
		break;
	case CMD_ENCRYPTED_SHINGLE:
		up_len = sizeof(session->cmd);
		encrypted = TRUE;
		is_shingle = TRUE;
		send_flags |= RSPAMD_FUZZY_REPLY_SHINGLE | RSPAMD_FUZZY_REPLY_ENCRYPTED;
		break;
	default:
		msg_err("invalid command type: %d", session->cmd_type);
		return;
	}

	memset(&result, 0, sizeof(result));
	memcpy(result.digest, cmd->digest, sizeof(result.digest));
	result.v1.flag = cmd->flag;
	result.v1.tag = cmd->tag;

	if (session->ctx->lua_pre_handlers != NULL) {
		struct rspamd_lua_fuzzy_script *cur;

		LL_FOREACH(session->ctx->lua_pre_handlers, cur)
		{
			/* Start lua pre handler */
			lua_State *L = session->ctx->cfg->lua_state;
			int err_idx, ret, nargs = 8;

			lua_pushcfunction(L, &rspamd_lua_traceback);
			err_idx = lua_gettop(L);
			/* Preallocate stack (small opt) */
			lua_checkstack(L, err_idx + nargs + 1);
			/* function */
			lua_rawgeti(L, LUA_REGISTRYINDEX, cur->cbref);
			/* client IP */
			rspamd_lua_ip_push(L, session->addr);
			/* client command */
			lua_pushinteger(L, cmd->cmd);
			/* command value (push as rspamd_text) */
			(void) lua_new_text(L, cmd->digest, sizeof(cmd->digest), FALSE);
			/* is shingle */
			lua_pushboolean(L, is_shingle);
			/* TODO: add additional data maybe (encryption, pubkey, etc) */
			rspamd_fuzzy_extensions_tolua(L, session);

			/* We push shingles merely for commands that modify content to avoid extra work */
			if (is_shingle && cmd->cmd != FUZZY_CHECK) {
				lua_newshingle(L, &session->cmd.sgl);
			}
			else {
				lua_pushnil(L);
			}

			/* Flag and value */
			lua_pushinteger(L, cmd->flag);
			lua_pushinteger(L, cmd->value);

			if ((ret = lua_pcall(L, nargs, LUA_MULTRET, err_idx)) != 0) {
				msg_err("call to lua_pre_handler lua "
						"script failed (%d): %s",
						ret, lua_tostring(L, -1));
			}
			else {
				/* Return values order:
				 * the first reply will be on err_idx + 1
				 * if it is true, then we need to read the former ones:
				 * 2-nd will be reply code
				 * 3-rd will be probability (or 0.0 if missing)
				 */
				ret = lua_toboolean(L, err_idx + 1);

				if (ret) {
					/* Artificial reply */
					result.v1.value = lua_tointeger(L, err_idx + 2);

					if (lua_isnumber(L, err_idx + 3)) {
						result.v1.prob = lua_tonumber(L, err_idx + 3);
					}
					else {
						result.v1.prob = 0.0f;
					}

					lua_settop(L, 0);
					rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);

					return;
				}
			}

			lua_settop(L, 0);
		}
	}


	if (G_UNLIKELY(cmd == NULL || up_len == 0)) {
		result.v1.value = 500;
		result.v1.prob = 0.0f;
		rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
		return;
	}

	if (session->ctx->encrypted_only && !encrypted) {
		/* Do not accept unencrypted commands */
		result.v1.value = 415;
		result.v1.prob = 0.0f;
		rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
		return;
	}

	/*
	 * UDP sessions were already screened before parsing, so re-checking here
	 * would just repeat the radix lookups on every accepted datagram. TCP
	 * command sessions still need it: the connection is only screened at
	 * accept time, and a dynamic block can land while it is open.
	 */
	if (!session->client_checked) {
		int block_code = rspamd_fuzzy_check_client(session->ctx, session->addr);
		if (block_code > 0) {
			session->ctx->stat.blocked_requests++;
			result.v1.value = block_code;
			result.v1.prob = 0.0f;
			rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
			return;
		}
	}

	if (session->key && session->addr) {
		ip_stat = rspamd_lru_hash_lookup(session->key->stat->last_ips,
										 session->addr, -1);

		if (ip_stat == NULL) {
			naddr = rspamd_inet_address_copy(session->addr, NULL);
			ip_stat = g_malloc0(sizeof(*ip_stat));
			REF_INIT_RETAIN(ip_stat, fuzzy_key_stat_dtor);
			rspamd_lru_hash_insert(session->key->stat->last_ips,
								   naddr, ip_stat, -1, 0);
		}

		REF_RETAIN(ip_stat);
		session->ip_stat = ip_stat;
	}
	else if (session->addr) {
		/*
		 * Unkeyed client (e.g. allowed by IP): track its traffic under the
		 * dedicated bucket, otherwise such writers are invisible in the stats
		 */
		if (session->ctx->unkeyed_stat == NULL) {
			struct fuzzy_key_stat *unkeyed = g_malloc0(sizeof(*unkeyed));

			REF_INIT_RETAIN(unkeyed, fuzzy_key_stat_dtor);
			unkeyed->last_ips = rspamd_lru_hash_new_full(1024,
														 (GDestroyNotify) rspamd_inet_address_free,
														 fuzzy_key_stat_unref,
														 rspamd_inet_address_hash,
														 rspamd_inet_address_equal);
			session->ctx->unkeyed_stat = unkeyed;
		}

		ip_stat = rspamd_lru_hash_lookup(session->ctx->unkeyed_stat->last_ips,
										 session->addr, -1);

		if (ip_stat == NULL) {
			naddr = rspamd_inet_address_copy(session->addr, NULL);
			ip_stat = g_malloc0(sizeof(*ip_stat));
			REF_INIT_RETAIN(ip_stat, fuzzy_key_stat_dtor);
			rspamd_lru_hash_insert(session->ctx->unkeyed_stat->last_ips,
								   naddr, ip_stat, -1, 0);
		}

		REF_RETAIN(ip_stat);
		session->ip_stat = ip_stat;
	}

	/*
	 * The per-source bucket covers every command that produces a reply
	 * without needing write authorisation: CHECK, PING and STAT. PING and
	 * STAT are answered unauthenticated, so leaving them unmetered let a
	 * source spend our parse and reply budget for free. Writes and deletes
	 * are gated by rspamd_fuzzy_check_write instead.
	 *
	 * This is inert unless ratelimit_rate and ratelimit_burst are configured,
	 * and rspamd_fuzzy_check_ratelimit already exempts ratelimit_whitelist
	 * and local addresses, which is where monitoring probes belong.
	 */
	bool is_rate_allowed = true;

	if (session->ctx->ratelimit_buckets &&
		(cmd->cmd == FUZZY_CHECK || cmd->cmd == FUZZY_PING ||
		 cmd->cmd == FUZZY_STAT)) {
		if (session->ctx->ratelimit_log_only) {
			(void) rspamd_fuzzy_check_ratelimit(session->ctx, session->addr,
												session->worker, session->timestamp); /* Check but ignore */
		}
		else {
			is_rate_allowed = rspamd_fuzzy_check_ratelimit(session->ctx, session->addr,
														   session->worker, session->timestamp);
		}
	}

	if (cmd->cmd == FUZZY_CHECK) {
		if (session->key && session->key->rl_bucket) {
			/* Check per-key bucket */

			enum rspamd_ratelimit_check_result res = rspamd_fuzzy_check_ratelimit_bucket(session->ctx, session->addr,
																						 session->timestamp, session->key->rl_bucket,
																						 ratelimit_policy_normal,
																						 session->key->burst,
																						 session->key->rate);

			if (res == ratelimit_new) {
				msg_info("ratelimiting key %s %.1f max elts",
						 session->key->name ? session->key->name : "unknown",
						 session->key->burst);

				struct rspamd_srv_command srv_cmd;

				srv_cmd.type = RSPAMD_SRV_FUZZY_BLOCKED;
				srv_cmd.cmd.fuzzy_blocked.af = rspamd_inet_address_get_af(session->addr);

				if (srv_cmd.cmd.fuzzy_blocked.af == AF_INET || srv_cmd.cmd.fuzzy_blocked.af == AF_INET6) {
					socklen_t slen;
					struct sockaddr *sa = rspamd_inet_address_get_sa(session->addr, &slen);

					if (slen <= sizeof(srv_cmd.cmd.fuzzy_blocked.addr)) {
						memcpy(&srv_cmd.cmd.fuzzy_blocked.addr, sa, slen);
						msg_debug("propagating blocked address to other workers");
						rspamd_srv_send_command(session->worker,
												session->ctx->event_loop,
												&srv_cmd, -1, NULL, NULL);
					}
					else {
						msg_err("bad address length: %d, expected to be %d",
								(int) slen, (int) sizeof(srv_cmd.cmd.fuzzy_blocked.addr));
					}
				}

				if (session->ctx->lua_blacklist_handlers) {
					struct rspamd_ratelimit_callback_ctx cb_ctx = {
						.addr = session->addr,
						.reason = "ratelimit",
						.type = RATELIMIT_EVENT_NEW,
						.bucket = session->key->rl_bucket,
						.max_burst = session->key->burst,
						.max_rate = session->key->rate,
						.session = session,
					};
					rspamd_fuzzy_call_ratelimit_handlers(session->ctx, &cb_ctx);
				}
				is_rate_allowed = session->ctx->ratelimit_log_only ? true : false;
			}
			else if (res == ratelimit_existing) {
				if (session->ctx->lua_blacklist_handlers) {
					struct rspamd_ratelimit_callback_ctx cb_ctx = {
						.addr = session->addr,
						.reason = "ratelimit",
						.type = RATELIMIT_EVENT_EXISTING,
						.bucket = session->key->rl_bucket,
						.max_burst = session->key->burst,
						.max_rate = session->key->rate,
						.session = session,
					};
					rspamd_fuzzy_call_ratelimit_handlers(session->ctx, &cb_ctx);
				}
				is_rate_allowed = session->ctx->ratelimit_log_only ? true : false;
			}
		}

		if (session->key && !isnan(session->key->expire)) {
			/* Check expire */
			static ev_tstamp today = NAN;

			/*
			 * Update `today` sometimes
			 */
			if (isnan(today)) {
				today = ev_time();
			}
			else if (rspamd_random_uint64_fast() > 0xFFFF000000000000ULL) {
				today = ev_time();
			}

			if (today > session->key->expire) {
				if (!session->key->expired) {
					msg_info("key %s is expired", session->key->name);
					session->key->expired = true;
				}

				result.v1.value = 503;
				result.v1.prob = 0.0f;
				rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
				return;
			}
		}

		/* Key is not allowed to read */
		if (session->key && !(session->key->flags & FUZZY_KEY_READ)) {
			result.v1.value = 503;
			result.v1.prob = 0.0f;
			rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
			return;
		}

		if (is_rate_allowed) {
			REF_RETAIN(session);
			rspamd_fuzzy_backend_check(session->ctx->backend, cmd,
									   rspamd_fuzzy_check_callback, session);
		}
		else {
			/* Should be 429 but we keep compatibility */
			result.v1.value = 403;
			result.v1.prob = 0.0f;
			result.v1.flag = 0;
			rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
		}
	}
	else if (cmd->cmd == FUZZY_STAT || cmd->cmd == FUZZY_PING) {
		/*
		 * Unlike CHECK, a rate limited PING or STAT is dropped rather than
		 * answered with 403: the reply *is* the entire cost of these
		 * commands, so replying anyway would leave egress unchanged and
		 * defeat the limit. This matches how a blocklisted source is handled.
		 */
		if (!is_rate_allowed) {
			session->ctx->stat.ratelimited_requests++;
			msg_debug("dropping ratelimited %s from %s",
					  cmd->cmd == FUZZY_PING ? "ping" : "stat",
					  rspamd_inet_address_to_string(session->addr));
			return;
		}

		if (cmd->cmd == FUZZY_STAT) {
			/* Store approximation (if needed) */
			result.v1.prob = session->ctx->stat.fuzzy_hashes;
			/* Store high qword in value and low qword in flag */
			result.v1.value = (int32_t) ((uint64_t) session->ctx->stat.fuzzy_hashes >> 32);
			result.v1.flag = (uint32_t) (session->ctx->stat.fuzzy_hashes & G_MAXUINT32);
		}
		else {
			result.v1.prob = 1.0f;
			result.v1.value = cmd->value;
		}

		rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
	}
	else {
		if (rspamd_fuzzy_check_write(session->ctx, session->addr, session->key, cmd->cmd)) {
			/* Check whitelist */
			if (session->ctx->skip_hashes && cmd->cmd == FUZZY_WRITE) {
				rspamd_encode_hex_buf(cmd->digest, sizeof(cmd->digest),
									  hexbuf, sizeof(hexbuf) - 1);
				hexbuf[sizeof(hexbuf) - 1] = '\0';

				if (rspamd_match_hash_map(session->ctx->skip_hashes,
										  hexbuf, sizeof(hexbuf) - 1)) {
					result.v1.value = 401;
					result.v1.prob = 0.0f;

					goto reply;
				}
			}

			if (session->ctx->weak_ids && kh_get(fuzzy_key_ids_set, session->ctx->weak_ids, cmd->flag) != kh_end(session->ctx->weak_ids)) {
				/* Flag command as weak */
				cmd->version |= RSPAMD_FUZZY_FLAG_WEAK;
			}

			/* Noop backends must skip all updates logic as irrelevant */
			if (!rspamd_fuzzy_backend_is_noop(session->ctx->backend)) {
				if (session->worker->index == 0 || session->ctx->peer_fd == -1) {
					/* Just add to the queue */
					up_cmd.is_shingle = is_shingle;
					ptr = is_shingle ? (gpointer) &up_cmd.cmd.shingle : (gpointer) &up_cmd.cmd.normal;
					memcpy(ptr, cmd, up_len);
					g_array_append_val(session->ctx->updates_pending, up_cmd);
				}
				else {
					/* We need to send request to the peer */
					up_req = g_malloc0(sizeof(*up_req));
					up_req->cmd.is_shingle = is_shingle;
					ptr = is_shingle ? (gpointer) &up_req->cmd.cmd.shingle : (gpointer) &up_req->cmd.cmd.normal;
					memcpy(ptr, cmd, up_len);

					enum fuzzy_peer_send_status sst = fuzzy_peer_try_send(
						session->ctx->peer_fd, up_req);

					if (sst == FUZZY_PEER_SEND_DONE) {
						g_free(up_req);
					}
					else if (sst == FUZZY_PEER_SEND_AGAIN) {
						up_req->ctx = session->ctx;
						DL_APPEND(session->ctx->pending_peer_requests, up_req);
						up_req->io_ev.data = up_req;
						ev_io_init(&up_req->io_ev, fuzzy_peer_send_io,
								   session->ctx->peer_fd, EV_WRITE);
						ev_io_start(session->ctx->event_loop, &up_req->io_ev);
					}
					else {
						msg_err("cannot send update request to the peer: %s",
								strerror(errno));
						g_free(up_req);
					}
				}
			}

			result.v1.value = 0;
			result.v1.prob = 1.0f;
		}
		else {
			result.v1.value = 503;
			result.v1.prob = 0.0f;
		}
	reply:
		rspamd_fuzzy_make_reply(cmd, &result, NULL, session, send_flags);
	}
}


static enum rspamd_fuzzy_epoch
rspamd_fuzzy_command_valid(struct rspamd_fuzzy_cmd *cmd, int r)
{
	enum rspamd_fuzzy_epoch ret = RSPAMD_FUZZY_EPOCH_MAX;

	switch (cmd->version & RSPAMD_FUZZY_VERSION_MASK) {
	case 5:
		if (cmd->shingles_count > 0) {
			if (r >= sizeof(struct rspamd_fuzzy_shingle_cmd)) {
				ret = RSPAMD_FUZZY_EPOCH12;
			}
		}
		else {
			if (r >= sizeof(*cmd)) {
				ret = RSPAMD_FUZZY_EPOCH12;
			}
		}
		break;
	case 4:
		if (cmd->shingles_count > 0) {
			if (r >= sizeof(struct rspamd_fuzzy_shingle_cmd)) {
				ret = (cmd->version & RSPAMD_FUZZY_V2_CAP) ? RSPAMD_FUZZY_EPOCH12 : RSPAMD_FUZZY_EPOCH11;
			}
		}
		else {
			if (r >= sizeof(*cmd)) {
				ret = (cmd->version & RSPAMD_FUZZY_V2_CAP) ? RSPAMD_FUZZY_EPOCH12 : RSPAMD_FUZZY_EPOCH11;
			}
		}
		break;
	case 3:
		if (cmd->shingles_count > 0) {
			if (r == sizeof(struct rspamd_fuzzy_shingle_cmd)) {
				ret = RSPAMD_FUZZY_EPOCH10;
			}
		}
		else {
			if (r == sizeof(*cmd)) {
				ret = RSPAMD_FUZZY_EPOCH10;
			}
		}
		break;
	default:
		break;
	}

	return ret;
}

static gboolean
rspamd_fuzzy_decrypt_command(struct fuzzy_session *s, unsigned char *buf, gsize buflen)
{
	struct rspamd_fuzzy_encrypted_req_hdr hdr;
	struct rspamd_cryptobox_pubkey *rk;
	struct fuzzy_key *key = NULL;

	if (s->ctx->default_key == NULL && s->ctx->dynamic_keys == NULL) {
		msg_warn("received encrypted request when encryption is not enabled");
		return FALSE;
	}

	if (buflen < sizeof(hdr)) {
		msg_warn("XXX: should not be reached");
		return FALSE;
	}

	memcpy(&hdr, buf, sizeof(hdr));
	buf += sizeof(hdr);
	buflen -= sizeof(hdr);

	/* Try to find the desired key */
	khiter_t k = kh_get(rspamd_fuzzy_keys_hash, s->ctx->keys, hdr.key_id);
	if (k == kh_end(s->ctx->keys)) {

		key = s->ctx->default_key;

		/* Check dynamic keys */
		if (s->ctx->dynamic_keys) {
			k = kh_get(rspamd_fuzzy_keys_hash, s->ctx->dynamic_keys, hdr.key_id);

			if (k != kh_end(s->ctx->dynamic_keys)) {
				key = kh_val(s->ctx->dynamic_keys, k);
			}
		}
	}
	else {
		key = kh_val(s->ctx->keys, k);
	}

	if (key == NULL) {
		/* Cannot find any suitable decryption key */
		msg_debug("cannot find suitable decryption key");
		return FALSE;
	}

	/* Now process the remote pubkey */
	rk = rspamd_pubkey_from_bin(hdr.pubkey, sizeof(hdr.pubkey), RSPAMD_KEYPAIR_KEX);

	if (rk == NULL) {
		/*
		 * Debug level on purpose: this is reachable by any host that can
		 * send us a datagram, before any rate limit applies, so logging it
		 * at error level turns a spoofed-source flood into one log line per
		 * packet. The decrypt_errors counter carries the signal instead.
		 */
		s->ctx->stat.decrypt_errors++;
		msg_debug("bad key; ip=%s",
				  rspamd_inet_address_to_string(s->addr));
		return FALSE;
	}

	/* Try to get the cached NM */
	rspamd_keypair_cache_process(s->ctx->keypair_cache, key->key, rk);

	/* Now decrypt request */
	if (!rspamd_cryptobox_decrypt_nm_inplace(buf, buflen, hdr.nonce,
											 rspamd_pubkey_get_nm(rk, key->key),
											 hdr.mac)) {
		/* Debug level for the same reason as the bad key case above */
		s->ctx->stat.decrypt_errors++;
		msg_debug("decryption failed; ip=%s",
				  rspamd_inet_address_to_string(s->addr));
		rspamd_pubkey_unref(rk);

		return FALSE;
	}

	s->key = key;
	REF_RETAIN(key);

	memcpy(s->nm, rspamd_pubkey_get_nm(rk, key->key), sizeof(s->nm));
	rspamd_pubkey_unref(rk);

	return TRUE;
}


static gboolean
rspamd_fuzzy_cmd_from_wire(unsigned char *buf, unsigned int buflen, struct fuzzy_session *s)
{
	enum rspamd_fuzzy_epoch epoch;
	gboolean encrypted = FALSE;

	if (buflen < sizeof(struct rspamd_fuzzy_cmd)) {
		msg_debug("truncated fuzzy command of size %d received", buflen);
		return FALSE;
	}

	/* Now check encryption */

	if (buflen >= sizeof(struct rspamd_fuzzy_encrypted_cmd)) {
		if (memcmp(buf, fuzzy_encrypted_magic, sizeof(fuzzy_encrypted_magic)) == 0) {
			/* Encrypted command */
			encrypted = TRUE;
		}
	}

	if (encrypted) {
		/* Decrypt first */
		if (!rspamd_fuzzy_decrypt_command(s, buf, buflen)) {
			return FALSE;
		}
		else {
			/*
			 * Advance buffer to skip encrypted header.
			 * Note that after rspamd_fuzzy_decrypt_command buf is unencrypted
			 */
			buf += sizeof(struct rspamd_fuzzy_encrypted_req_hdr);
			buflen -= sizeof(struct rspamd_fuzzy_encrypted_req_hdr);
		}
	}

	/* Fill the normal command */
	if (buflen < sizeof(s->cmd.basic)) {
		msg_debug("truncated normal fuzzy command of size %d received", buflen);
		return FALSE;
	}

	memcpy(&s->cmd.basic, buf, sizeof(s->cmd.basic));
	epoch = rspamd_fuzzy_command_valid(&s->cmd.basic, buflen);

	if (epoch == RSPAMD_FUZZY_EPOCH_MAX) {
		msg_debug("invalid fuzzy command of size %d received", buflen);
		return FALSE;
	}

	s->epoch = epoch;

	/* Advance buf */
	buf += sizeof(s->cmd.basic);
	buflen -= sizeof(s->cmd.basic);

	if (s->cmd.basic.shingles_count > 0) {
		if (buflen >= sizeof(s->cmd.sgl)) {
			/* Copy the shingles part */
			memcpy(&s->cmd.sgl, buf, sizeof(s->cmd.sgl));
		}
		else {
			/* Truncated stuff */
			msg_debug("truncated fuzzy shingles command of size %d received", buflen);
			return FALSE;
		}

		buf += sizeof(s->cmd.sgl);
		buflen -= sizeof(s->cmd.sgl);

		if (encrypted) {
			s->cmd_type = CMD_ENCRYPTED_SHINGLE;
		}
		else {
			s->cmd_type = CMD_SHINGLE;
		}
	}
	else {
		if (encrypted) {
			s->cmd_type = CMD_ENCRYPTED_NORMAL;
		}
		else {
			s->cmd_type = CMD_NORMAL;
		}
	}

	if (buflen > 0) {
		/* Process possible extensions */
		if (!rspamd_fuzzy_extensions_from_wire(buf, buflen, &s->extensions)) {
			msg_debug("malformed extensions in a fuzzy command of size %d received",
					  buflen);
			return FALSE;
		}
	}

	return TRUE;
}


static void
fuzzy_session_destroy(gpointer d)
{
	struct fuzzy_session *session = d;

	/* Stop IO watcher if active (e.g. pending UDP write retry) */
	if (ev_can_stop(&session->io)) {
		ev_io_stop(session->ctx->event_loop, &session->io);
	}

	rspamd_inet_address_free(session->addr);
	rspamd_explicit_memzero(session->nm, sizeof(session->nm));

	/* Only decrement nconns if this is not a TCP command session */
	if (session->tcp_session == NULL) {
		session->worker->nconns--;
	}
	else {
		/* Release the reference to the TCP session */
		REF_RELEASE(session->tcp_session);
	}

	if (session->ip_stat) {
		REF_RELEASE(session->ip_stat);
	}

	if (session->extensions) {
		g_free(session->extensions);
	}

	if (session->key) {
		REF_RELEASE(session->key);
	}

	g_free(session);
}

static void
fuzzy_tcp_session_destroy(gpointer d)
{
	struct fuzzy_tcp_session *session = d;

	msg_debug_fuzzy_storage("destroying TCP session from %s",
							rspamd_inet_address_to_string(session->common.addr));

	/*
	 * Normally rspamd_fuzzy_tcp_session_close has already done all of this;
	 * these are kept as a safety net for a session that is destroyed without
	 * ever having been closed.
	 */
	if (ev_can_stop(&session->common.io)) {
		ev_io_stop(session->common.ctx->event_loop, &session->common.io);
	}

	if (ev_can_stop(&session->tm)) {
		ev_timer_stop(session->common.ctx->event_loop, &session->tm);
	}

	/* Free replies queue */
	struct fuzzy_tcp_reply_queue_elt *elt, *tmp;
	DL_FOREACH_SAFE(session->replies_queue, elt, tmp)
	{
		DL_DELETE(session->replies_queue, elt);
		g_free(elt);
	}

	if (session->common.fd != -1) {
		close(session->common.fd);
		session->common.fd = -1;
	}

	rspamd_inet_address_free(session->common.addr);
	session->common.worker->nconns--;

	g_free(session);
}

#define FUZZY_INPUT_BUFLEN 1024
#ifdef HAVE_RECVMMSG
#define MSGVEC_LEN 16
#else
#define MSGVEC_LEN 1
#endif

union sa_union {
	struct sockaddr sa;
	struct sockaddr_in s4;
	struct sockaddr_in6 s6;
	struct sockaddr_un su;
	struct sockaddr_storage ss;
};
/*
 * Accept new connection and construct task
 */
static void
accept_fuzzy_socket(EV_P_ ev_io *w, int revents)
{
	struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
	struct rspamd_fuzzy_storage_ctx *ctx;
	struct fuzzy_session *session;
	gssize r, msg_len;
	uint64_t *nerrors;
	struct iovec iovs[MSGVEC_LEN];
	uint8_t bufs[MSGVEC_LEN][FUZZY_INPUT_BUFLEN];
	union sa_union peer_sa[MSGVEC_LEN];
	socklen_t salen = sizeof(peer_sa[0]);
#ifdef HAVE_RECVMMSG
#define MSG_FIELD(msg, field) msg.msg_hdr.field
	struct mmsghdr msg[MSGVEC_LEN];
#else
#define MSG_FIELD(msg, field) msg.field
	struct msghdr msg[MSGVEC_LEN];
#endif

	memset(msg, 0, sizeof(*msg) * MSGVEC_LEN);
	ctx = (struct rspamd_fuzzy_storage_ctx *) worker->ctx;

	/* Prepare messages to receive */
	for (int i = 0; i < MSGVEC_LEN; i++) {
		/* Prepare msghdr structs */
		iovs[i].iov_base = bufs[i];
		iovs[i].iov_len = sizeof(bufs[i]);
		MSG_FIELD(msg[i], msg_name) = (void *) &peer_sa[i];
		MSG_FIELD(msg[i], msg_namelen) = salen;
		MSG_FIELD(msg[i], msg_iov) = &iovs[i];
		MSG_FIELD(msg[i], msg_iovlen) = 1;
	}

	/* Got some data */
	if (revents == EV_READ) {
		ev_now_update_if_cheap(ctx->event_loop);
		for (;;) {
			/*
			 * The kernel overwrites msg_namelen on each recvmsg/recvmmsg call
			 * with the actual size of the received source address. If we don't
			 * reset it back to the buffer capacity before the next call, the
			 * kernel will treat that smaller value as the buffer size and
			 * truncate a larger incoming address (e.g. IPv6 after IPv4),
			 * leaving stale stack bytes mixed into the parsed sockaddr.
			 */
			for (int i = 0; i < MSGVEC_LEN; i++) {
				MSG_FIELD(msg[i], msg_namelen) = salen;
			}
#ifdef HAVE_RECVMMSG
			r = recvmmsg(w->fd, msg, MSGVEC_LEN, 0, NULL);
#else
			r = recvmsg(w->fd, msg, 0);
#endif

			if (r == -1) {
				if (errno == EINTR) {
					continue;
				}
				else if (errno == EAGAIN || errno == EWOULDBLOCK) {

					return;
				}

				msg_err("got error while reading from socket: %d, %s",
						errno,
						strerror(errno));
				return;
			}

#ifndef HAVE_RECVMMSG
			msg_len = r; /* Save real length in bytes here */
			r = 1;       /* Assume that we have received a single message */
#endif

			for (int i = 0; i < r; i++) {
				rspamd_inet_addr_t *client_addr;

				if (MSG_FIELD(msg[i], msg_namelen) >= sizeof(struct sockaddr)) {
					client_addr = rspamd_inet_address_from_sa(MSG_FIELD(msg[i], msg_name),
															  MSG_FIELD(msg[i], msg_namelen));
				}
				else {
					client_addr = NULL;
				}

				/*
				 * Drop blocklisted sources before doing any work on the
				 * datagram. This only needs the source address, so a blocked
				 * peer costs one radix lookup instead of a session
				 * allocation, command parsing, an ECDH plus MAC verification
				 * for encrypted commands, and the Lua pre-handlers.
				 *
				 * Nothing is sent back: building a reply would require
				 * parsing the command first, which is exactly the work being
				 * avoided. The TCP path already drops blocked peers without
				 * a reply (see accept_tcp_socket), so this makes the two
				 * transports behave alike.
				 */
				if (client_addr && rspamd_fuzzy_check_client(ctx, client_addr) > 0) {
					ctx->stat.blocked_requests++;
					msg_debug("dropping fuzzy command from blocked address %s",
							  rspamd_inet_address_to_string(client_addr));
					rspamd_inet_address_free(client_addr);
					continue;
				}

				session = g_malloc0(sizeof(*session));
				REF_INIT_RETAIN(session, fuzzy_session_destroy);
				session->worker = worker;
				session->fd = w->fd;
				session->ctx = ctx;
				session->timestamp = ev_now(ctx->event_loop);
				session->addr = client_addr;
				session->client_checked = true;
				worker->nconns++;

				/* Each message can have its length in case of recvmmsg */
#ifdef HAVE_RECVMMSG
				msg_len = msg[i].msg_len;
#endif

				if (rspamd_fuzzy_cmd_from_wire(iovs[i].iov_base,
											   msg_len, session)) {
					/* Check shingles count sanity */
					rspamd_fuzzy_process_command(session);
				}
				else {
					/* Discard input */
					session->ctx->stat.invalid_requests++;
					msg_debug("invalid fuzzy command of size %z received", r);

					/*
					 * errors_ips is telemetry only: it is reported by
					 * fuzzystat and deliberately never feeds a blocking
					 * decision. It is incremented exactly when parsing
					 * failed, which takes no key and no handshake, so on UDP
					 * the source address here is trivially forgeable. Banning
					 * on it would let anyone silence an arbitrary third party
					 * by sending a handful of malformed datagrams carrying
					 * that victim's address. Operators who know their network
					 * is spoof resistant can implement their own policy with
					 * worker:block_fuzzy_client().
					 */
					if (session->addr) {
						nerrors = rspamd_lru_hash_lookup(session->ctx->errors_ips,
														 session->addr, -1);

						if (nerrors == NULL) {
							nerrors = g_malloc(sizeof(*nerrors));
							*nerrors = 1;
							rspamd_lru_hash_insert(session->ctx->errors_ips,
												   rspamd_inet_address_copy(session->addr, NULL),
												   nerrors, -1, -1);
						}
						else {
							*nerrors = *nerrors + 1;
						}
					}
				}

				REF_RELEASE(session);
			}
#ifdef HAVE_RECVMMSG
			/* Stop reading as we are using recvmmsg instead of recvmsg */
			break;
#endif
		}
	}
}

/* TCP-specific reply and I/O handlers */

/*
 * Terminate a TCP connection exactly once.
 *
 * The session object is refcounted and shared with every in-flight command
 * session, so reaching a terminal condition on the socket cannot simply
 * REF_RELEASE: outstanding commands keep the refcount above zero, the
 * destructor does not run, and the watchers would stay armed. The next
 * readiness event (EOF is permanently readable) or the next tick of the
 * repeating timeout would then release the owner reference again, dropping
 * the refcount below what the in-flight commands actually hold and freeing
 * the session while their callbacks still point at it.
 *
 * So do the teardown here instead: disarm both watchers, close the socket,
 * mark the session closed, and drop the single owner reference — all of it
 * guarded by the `closed` flag so repeated calls are harmless. Whatever
 * commands are still in flight keep the object alive and their replies get
 * discarded by rspamd_fuzzy_tcp_enqueue_reply.
 */
static void
rspamd_fuzzy_tcp_session_close(struct fuzzy_tcp_session *session)
{
	if (session->closed) {
		return;
	}

	session->closed = true;

	if (ev_can_stop(&session->common.io)) {
		ev_io_stop(session->common.ctx->event_loop, &session->common.io);
	}

	if (ev_can_stop(&session->tm)) {
		ev_timer_stop(session->common.ctx->event_loop, &session->tm);
	}

	/*
	 * Release the descriptor now rather than waiting for the last in-flight
	 * command to complete, and poison it so no late path can write to a fd
	 * number that has been recycled by another connection.
	 */
	if (session->common.fd != -1) {
		close(session->common.fd);
		session->common.fd = -1;
	}

	/* Drop the reference taken by accept_tcp_socket */
	REF_RELEASE(session);
}

static void
rspamd_fuzzy_tcp_timeout(EV_P_ ev_timer *w, int revents)
{
	struct fuzzy_tcp_session *session = (struct fuzzy_tcp_session *) w->data;

	msg_debug_fuzzy_storage("TCP session from %s timed out",
							rspamd_inet_address_to_string(session->common.addr));

	rspamd_fuzzy_tcp_session_close(session);
}

static enum fuzzy_tcp_write_status
rspamd_fuzzy_tcp_write_reply(struct fuzzy_tcp_session *session,
							 struct fuzzy_tcp_reply_queue_elt *reply)
{
	gssize r;
	gsize total_len = sizeof(reply->rep.size_hdr) + GUINT16_FROM_LE(reply->rep.size_hdr);
	gsize remaining = total_len - reply->written;
	unsigned char *data = ((unsigned char *) &reply->rep) + reply->written;

	r = write(session->common.fd, data, remaining);

	if (r == -1) {
		if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) {
			return FUZZY_TCP_WRITE_AGAIN;
		}
		else {
			msg_err("error while writing TCP reply to %s: %s",
					rspamd_inet_address_to_string(session->common.addr),
					strerror(errno));

			return FUZZY_TCP_WRITE_FATAL;
		}
	}

	reply->written += r;

	if (reply->written >= total_len) {
		/* Reply fully sent */
		DL_DELETE(session->replies_queue, reply);
		g_free(reply);

		msg_debug_fuzzy_storage("TCP reply sent to %s, %z bytes",
								rspamd_inet_address_to_string(session->common.addr),
								(size_t) r);

		return FUZZY_TCP_WRITE_DONE;
	}

	return FUZZY_TCP_WRITE_AGAIN;
}

static void
rspamd_fuzzy_tcp_io(EV_P_ ev_io *w, int revents)
{
	struct fuzzy_tcp_session *session = (struct fuzzy_tcp_session *) w->data;
	gssize r;

	if (session->closed) {
		/* Should not happen: closing disarms this watcher */
		return;
	}

	if (revents & EV_READ) {
		/* Read available data */
		r = read(session->common.fd,
				 session->input_buf + session->bytes_unprocessed,
				 sizeof(session->input_buf) - session->bytes_unprocessed);

		if (r <= 0) {
			if (r == -1) {
				if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) {
					return;
				}
				msg_debug_fuzzy_storage("read error on TCP connection from %s: %s",
										rspamd_inet_address_to_string(session->common.addr),
										strerror(errno));
			}
			else {
				msg_debug_fuzzy_storage("TCP connection from %s closed by peer",
										rspamd_inet_address_to_string(session->common.addr));
			}

			rspamd_fuzzy_tcp_session_close(session);
			return;
		}

		session->bytes_unprocessed += r;
		session->common.timestamp = ev_now(session->common.ctx->event_loop);

		/* Reset timeout */
		ev_timer_again(EV_A_ & session->tm);

		/* Process frames */
		unsigned int processed_offset = 0;

		while (processed_offset < session->bytes_unprocessed) {
			uint16_t frame_len;

			/* Check frame state */
			if ((session->cur_frame_state & 0xC000) == 0x0000) {
				/* Need to read first byte of length */
				if (processed_offset < session->bytes_unprocessed) {
					session->cur_frame_state = 0x8000 | session->input_buf[processed_offset];
					processed_offset++;
				}
				else {
					break;
				}
			}

			if ((session->cur_frame_state & 0xC000) == 0x8000) {
				/* Need to read second byte of length */
				if (processed_offset < session->bytes_unprocessed) {
					uint16_t first_byte = session->cur_frame_state & 0xFF;
					uint16_t second_byte = session->input_buf[processed_offset];
					/* Reconstruct in little-endian byte order */
					uint16_t real_len = first_byte | (second_byte << 8);
					/*
					 * Length is stored in the low 14 bits of cur_frame_state
					 * (top two bits are state flags). FUZZY_TCP_BUFFER_LENGTH
					 * fits in 14 bits, so any value with bit 14 or 15 set is
					 * invalid and would otherwise be silently masked off,
					 * creating a protocol-smuggling primitive. Reject here.
					 */
					if (real_len == 0 || real_len > FUZZY_TCP_BUFFER_LENGTH) {
						msg_err("invalid frame length %d from %s, closing connection",
								(int) real_len,
								rspamd_inet_address_to_string(session->common.addr));
						rspamd_fuzzy_tcp_session_close(session);
						return;
					}
					session->cur_frame_state = 0xC000 | real_len;
					processed_offset++;
				}
				else {
					break;
				}
			}

			/* Now we have full length */
			frame_len = session->cur_frame_state & 0x3FFF;

			if (frame_len > FUZZY_TCP_BUFFER_LENGTH || frame_len == 0) {
				msg_err("invalid frame length %d from %s, closing connection",
						(int) frame_len,
						rspamd_inet_address_to_string(session->common.addr));
				rspamd_fuzzy_tcp_session_close(session);
				return;
			}

			/* Check if we have complete frame */
			if (session->bytes_unprocessed - processed_offset >= frame_len) {
				/*
				 * Per-frame command session. Owns its own key retain,
				 * extensions buffer, ip_stat retain, and decrypt nm —
				 * fuzzy_session_destroy releases all of that when this
				 * cmd_session is dropped. Do NOT cache any of it on the
				 * long-lived TCP session: that turned the per-frame
				 * allocations into per-frame leaks.
				 */
				struct fuzzy_session *cmd_session = g_malloc0(sizeof(*cmd_session));
				REF_INIT_RETAIN(cmd_session, fuzzy_session_destroy);

				cmd_session->worker = session->common.worker;
				cmd_session->addr = rspamd_inet_address_copy(session->common.addr, NULL);
				cmd_session->ctx = session->common.ctx;
				cmd_session->fd = session->common.fd;
				cmd_session->timestamp = session->common.timestamp;

				/* Replies go to the TCP queue, not the socket directly */
				cmd_session->tcp_session = session;
				REF_RETAIN(session); /* released by fuzzy_session_destroy */

				if (rspamd_fuzzy_cmd_from_wire(session->input_buf + processed_offset,
											   frame_len, cmd_session)) {
					rspamd_fuzzy_process_command(cmd_session);
				}
				else {
					session->common.ctx->stat.invalid_requests++;
					msg_debug_fuzzy_storage("invalid TCP fuzzy command of size %d received from %s",
											(int) frame_len,
											rspamd_inet_address_to_string(session->common.addr));
				}

				REF_RELEASE(cmd_session);

				processed_offset += frame_len;
				session->cur_frame_state = 0x0000; /* Reset for next frame */
			}
			else {
				/* Incomplete frame, wait for more data */
				break;
			}
		}

		/* Move unprocessed data to the beginning */
		if (processed_offset > 0) {
			if (processed_offset < session->bytes_unprocessed) {
				memmove(session->input_buf,
						session->input_buf + processed_offset,
						session->bytes_unprocessed - processed_offset);
				session->bytes_unprocessed -= processed_offset;
			}
			else {
				session->bytes_unprocessed = 0;
			}
		}
	}

	if (revents & EV_WRITE) {
		/* Write pending replies */
		struct fuzzy_tcp_reply_queue_elt *elt;

		while ((elt = session->replies_queue) != NULL) {
			enum fuzzy_tcp_write_status status;

			status = rspamd_fuzzy_tcp_write_reply(session, elt);

			if (status == FUZZY_TCP_WRITE_AGAIN) {
				/* Cannot write more, wait for next write event */
				return;
			}
			else if (status == FUZZY_TCP_WRITE_FATAL) {
				/*
				 * The socket is unusable; leaving the write watcher armed
				 * would just spin on a permanently writable error condition.
				 */
				rspamd_fuzzy_tcp_session_close(session);
				return;
			}
		}

		/* All replies sent, disable write event */
		ev_io_stop(EV_A_ w);
		ev_io_set(w, w->fd, EV_READ);
		ev_io_start(EV_A_ w);
	}
}

static void
accept_tcp_socket(EV_P_ ev_io *w, int revents)
{
	struct rspamd_worker *worker = (struct rspamd_worker *) w->data;
	struct rspamd_fuzzy_storage_ctx *ctx;
	struct fuzzy_tcp_session *session;
	rspamd_inet_addr_t *addr = NULL;
	int nfd;

	ctx = (struct rspamd_fuzzy_storage_ctx *) worker->ctx;

	if ((nfd = rspamd_accept_from_socket(w->fd, &addr,
										 rspamd_worker_throttle_accept_events, worker->accept_events)) == -1) {
		msg_warn("TCP accept failed: %s", strerror(errno));
		return;
	}

	/* Check for EAGAIN */
	if (nfd == 0) {
		if (addr) {
			rspamd_inet_address_free(addr);
		}
		return;
	}

	ev_now_update_if_cheap(ctx->event_loop);

	/* Check ratelimit */
	if (!rspamd_fuzzy_check_ratelimit(ctx, addr, worker, ev_now(ctx->event_loop))) {
		msg_info("ratelimiting TCP connection from %s",
				 rspamd_inet_address_to_string(addr));
		rspamd_inet_address_free(addr);
		close(nfd);
		return;
	}

	/* Check if client is allowed */
	int block_code = rspamd_fuzzy_check_client(ctx, addr);
	if (block_code > 0) {
		msg_info("refusing TCP connection from %s (blacklisted)",
				 rspamd_inet_address_to_string(addr));
		rspamd_inet_address_free(addr);
		close(nfd);
		return;
	}

	/* Set TCP_NODELAY */
#ifdef TCP_NODELAY
	{
		int sopt = 1;
		if (setsockopt(nfd, IPPROTO_TCP, TCP_NODELAY, &sopt, sizeof(sopt)) == -1) {
			msg_warn("cannot set TCP_NODELAY for %s: %s",
					 rspamd_inet_address_to_string(addr),
					 strerror(errno));
		}
	}
#endif

	/* Create session */
	session = g_malloc0(sizeof(*session));
	REF_INIT_RETAIN(session, fuzzy_tcp_session_destroy);

	session->common.ctx = ctx;
	session->common.worker = worker;
	session->common.addr = addr;
	session->common.fd = nfd;
	session->common.timestamp = ev_now(ctx->event_loop);

	session->cur_frame_state = 0x0000;
	session->bytes_unprocessed = 0;
	session->replies_queue = NULL;

	worker->nconns++;

	msg_debug_fuzzy_storage("accepted TCP connection from %s",
							rspamd_inet_address_to_string(addr));

	/* Setup I/O watcher */
	session->common.io.data = session;
	ev_io_init(&session->common.io, rspamd_fuzzy_tcp_io, nfd, EV_READ);
	ev_io_start(ctx->event_loop, &session->common.io);

	/* Setup timeout */
	session->tm.data = session;
	ev_timer_init(&session->tm, rspamd_fuzzy_tcp_timeout,
				  ctx->tcp_timeout, ctx->tcp_timeout);
	ev_timer_start(ctx->event_loop, &session->tm);
}

static gboolean
rspamd_fuzzy_storage_periodic_callback(void *ud)
{
	struct rspamd_fuzzy_storage_ctx *ctx = ud;

	if (ctx->updates_pending->len > 0) {
		rspamd_fuzzy_process_updates_queue(ctx, local_db_name, FALSE);

		return TRUE;
	}

	return FALSE;
}

static gboolean
rspamd_fuzzy_storage_sync(struct rspamd_main *rspamd_main,
						  struct rspamd_worker *worker, int fd,
						  int attached_fd,
						  struct rspamd_control_command *cmd,
						  gpointer ud)
{
	struct rspamd_fuzzy_storage_ctx *ctx = ud;
	struct rspamd_control_reply rep;

	memset(&rep, 0, sizeof(rep));
	rep.type = RSPAMD_CONTROL_FUZZY_SYNC;
	rep.id = cmd->id;
	rep.reply.fuzzy_sync.status = 0;

	if (ctx->backend && worker->index == 0) {
		rspamd_fuzzy_process_updates_queue(ctx, local_db_name, FALSE);
		rspamd_fuzzy_backend_start_update(ctx->backend, ctx->sync_timeout,
										  rspamd_fuzzy_storage_periodic_callback, ctx);
	}

	if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
		msg_err("cannot write reply to the control socket: %s",
				strerror(errno));
	}

	return TRUE;
}

static gboolean
rspamd_fuzzy_control_blocked(struct rspamd_main *rspamd_main,
							 struct rspamd_worker *worker, int fd,
							 int attached_fd,
							 struct rspamd_control_command *cmd,
							 gpointer ud)
{
	struct rspamd_fuzzy_storage_ctx *ctx = (struct rspamd_fuzzy_storage_ctx *) ud;
	struct rspamd_control_reply rep;
	struct rspamd_leaky_bucket_elt *elt;
	ev_tstamp now = ev_now(ctx->event_loop);
	rspamd_inet_addr_t *addr = NULL;

	memset(&rep, 0, sizeof(rep));
	rep.type = RSPAMD_CONTROL_FUZZY_BLOCKED;
	rep.id = cmd->id;
	rep.reply.fuzzy_blocked.status = 0;

	if (cmd->cmd.fuzzy_blocked.af == AF_INET) {
		addr = rspamd_inet_address_from_sa(&cmd->cmd.fuzzy_blocked.addr.sa,
										   sizeof(struct sockaddr_in));
	}
	else if (cmd->cmd.fuzzy_blocked.af == AF_INET6) {
		addr = rspamd_inet_address_from_sa(&cmd->cmd.fuzzy_blocked.addr.sa,
										   sizeof(struct sockaddr_in6));
	}
	else {
		msg_err("invalid address family: %d", cmd->cmd.fuzzy_blocked.af);
		rep.reply.fuzzy_blocked.status = -1;
	}

	if (addr) {
		elt = rspamd_lru_hash_lookup(ctx->ratelimit_buckets, addr,
									 (time_t) now);

		if (elt) {
			if (isnan(elt->cur)) {
				/* Already ratelimited, ignore */
			}
			else {
				elt->last = now;
				elt->cur = NAN;

				msg_info("propagating ratelimiting %s, %.1f max elts",
						 rspamd_inet_address_to_string(addr),
						 ctx->leaky_bucket_burst);
				if (ctx->lua_blacklist_handlers) {
					struct rspamd_ratelimit_callback_ctx cb_ctx = {
						.addr = addr,
						.reason = "ratelimit",
						.type = RATELIMIT_EVENT_NEW,
						.bucket = elt,
						.max_burst = ctx->leaky_bucket_burst,
						.max_rate = ctx->leaky_bucket_rate,
						.session = NULL,
					};
					rspamd_fuzzy_call_ratelimit_handlers(ctx, &cb_ctx);
				}
			}

			rspamd_inet_address_free(addr);
		}
		else {
			/* New bucket */
			elt = g_malloc(sizeof(*elt));
			elt->addr = addr; /* transfer ownership */
			elt->cur = NAN;
			elt->last = now;

			rspamd_lru_hash_insert(ctx->ratelimit_buckets,
								   addr,
								   elt,
								   (time_t) now,
								   ctx->leaky_bucket_ttl);
			msg_info("propagating ratelimiting %s, %.1f max elts",
					 rspamd_inet_address_to_string(addr),
					 ctx->leaky_bucket_burst);
			if (ctx->lua_blacklist_handlers) {
				struct rspamd_ratelimit_callback_ctx cb_ctx = {
					.addr = addr,
					.reason = "ratelimit",
					.type = RATELIMIT_EVENT_NEW,
					.bucket = elt,
					.max_burst = ctx->leaky_bucket_burst,
					.max_rate = ctx->leaky_bucket_rate,
					.session = NULL,
				};
				rspamd_fuzzy_call_ratelimit_handlers(ctx, &cb_ctx);
			}
		}
	}

	if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
		msg_err("cannot write reply to the control socket: %s",
				strerror(errno));
	}

	return TRUE;
}

static gboolean
rspamd_fuzzy_storage_reload(struct rspamd_main *rspamd_main,
							struct rspamd_worker *worker, int fd,
							int attached_fd,
							struct rspamd_control_command *cmd,
							gpointer ud)
{
	struct rspamd_fuzzy_storage_ctx *ctx = ud;
	GError *err = NULL;
	struct rspamd_control_reply rep;

	msg_info("reloading fuzzy storage after receiving reload command");

	if (ctx->backend) {
		/* Close backend and reopen it one more time */
		rspamd_fuzzy_backend_close(ctx->backend);
	}

	memset(&rep, 0, sizeof(rep));
	rep.type = RSPAMD_CONTROL_RELOAD;
	rep.id = cmd->id;

	if ((ctx->backend = rspamd_fuzzy_backend_create(ctx->event_loop,
													worker->cf->options, rspamd_main->cfg,
													&err)) == NULL) {
		msg_err("cannot open backend after reload: %e", err);
		rep.reply.reload.status = err->code;
		g_error_free(err);
	}
	else {
		rep.reply.reload.status = 0;
	}

	if (ctx->backend && worker->index == 0) {
		rspamd_fuzzy_backend_start_update(ctx->backend, ctx->sync_timeout,
										  rspamd_fuzzy_storage_periodic_callback, ctx);
	}

	if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
		msg_err("cannot write reply to the control socket: %s",
				strerror(errno));
	}

	return TRUE;
}

static int
lua_fuzzy_add_pre_handler(lua_State *L)
{
	struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
								   rspamd_lua_check_udata(L, 1, rspamd_worker_classname);
	struct rspamd_fuzzy_storage_ctx *ctx;

	if (!pwrk) {
		return luaL_error(L, "invalid arguments, worker + function are expected");
	}

	wrk = *pwrk;

	if (wrk && lua_isfunction(L, 2)) {
		ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx;
		struct rspamd_lua_fuzzy_script *script;

		script = g_malloc0(sizeof(*script));
		lua_pushvalue(L, 2);
		script->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
		LL_APPEND(ctx->lua_pre_handlers, script);
	}
	else {
		return luaL_error(L, "invalid arguments, worker + function are expected");
	}

	return 0;
}

static int
lua_fuzzy_add_post_handler(lua_State *L)
{
	struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
								   rspamd_lua_check_udata(L, 1, rspamd_worker_classname);
	struct rspamd_fuzzy_storage_ctx *ctx;

	if (!pwrk) {
		return luaL_error(L, "invalid arguments, worker + function are expected");
	}

	wrk = *pwrk;
	ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx;

	if (wrk && lua_isfunction(L, 2)) {
		ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx;
		struct rspamd_lua_fuzzy_script *script;

		script = g_malloc0(sizeof(*script));
		lua_pushvalue(L, 2);
		script->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
		LL_APPEND(ctx->lua_post_handlers, script);
	}
	else {
		return luaL_error(L, "invalid arguments, worker + function are expected");
	}

	return 0;
}

static int
lua_fuzzy_add_blacklist_handler(lua_State *L)
{
	struct rspamd_worker *wrk, **pwrk = (struct rspamd_worker **)
								   rspamd_lua_check_udata(L, 1, rspamd_worker_classname);
	struct rspamd_fuzzy_storage_ctx *ctx;

	if (!pwrk) {
		return luaL_error(L, "invalid arguments, worker + function are expected");
	}

	wrk = *pwrk;
	ctx = (struct rspamd_fuzzy_storage_ctx *) wrk->ctx;

	if (wrk && lua_isfunction(L, 2)) {
		struct rspamd_lua_fuzzy_script *script;

		script = g_malloc0(sizeof(*script));
		lua_pushvalue(L, 2);
		script->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
		LL_APPEND(ctx->lua_blacklist_handlers, script);
	}
	else {
		return luaL_error(L, "invalid arguments, worker + function are expected");
	}

	return 0;
}

/*
 * worker:block_fuzzy_client(addr_string, prefix_len[, expire_ts[, reason[, response_code]]])
 *
 * Dynamically block an IP address or CIDR network in this worker.
 * expire_ts is an absolute monotonic timestamp (rspamd_util.get_time() + ttl).
 * Pass nil or 0 for a permanent block (until worker restart).
 * response_code is the fuzzy reply code (default 503); use 403 for ratelimits.
 *
 * Returns: true on success; false, errmsg on parse failure.
 */

/*
 * worker:is_ratelimit_whitelisted(addr_string) -> bool
 * Returns true if addr is present in the fuzzy worker's ratelimit_whitelist map.
 * Returns false when the map is not configured or the address is not found.
 */
static int
lua_fuzzy_is_ratelimit_whitelisted(lua_State *L)
{
	struct rspamd_worker **pwrk = (struct rspamd_worker **)
		rspamd_lua_check_udata(L, 1, rspamd_worker_classname);

	if (!pwrk) {
		return luaL_error(L, "invalid self: worker expected");
	}

	struct rspamd_fuzzy_storage_ctx *ctx = (struct rspamd_fuzzy_storage_ctx *) (*pwrk)->ctx;
	size_t addr_len;
	const char *addr_str = luaL_checklstring(L, 2, &addr_len);

	if (ctx->ratelimit_whitelist == NULL) {
		lua_pushboolean(L, FALSE);
		return 1;
	}

	rspamd_inet_addr_t *addr;
	if (!rspamd_parse_inet_address(&addr, addr_str, addr_len,
								   RSPAMD_INET_ADDRESS_PARSE_NO_UNIX | RSPAMD_INET_ADDRESS_PARSE_NO_PORT)) {
		lua_pushboolean(L, FALSE);
		return 1;
	}

	gboolean whitelisted = rspamd_match_radix_map_addr(ctx->ratelimit_whitelist, addr) != NULL;
	rspamd_inet_address_free(addr);

	lua_pushboolean(L, whitelisted);
	return 1;
}

static int
lua_fuzzy_block_client(lua_State *L)
{
	struct rspamd_worker **pwrk = (struct rspamd_worker **)
		rspamd_lua_check_udata(L, 1, rspamd_worker_classname);

	if (!pwrk) {
		return luaL_error(L, "invalid self: worker expected");
	}

	struct rspamd_fuzzy_storage_ctx *ctx = (struct rspamd_fuzzy_storage_ctx *) (*pwrk)->ctx;

	const char *addr_str = luaL_checkstring(L, 2);
	unsigned int prefix_len = (unsigned int) luaL_checkinteger(L, 3);
	double expire_ts = luaL_optnumber(L, 4, 0.0);
	const char *reason = luaL_optstring(L, 5, "lua");
	int32_t response_code = (int32_t) luaL_optinteger(L, 6, 0);

	if (rspamd_fuzzy_block_addr(ctx, addr_str, prefix_len, expire_ts, reason, response_code)) {
		lua_pushboolean(L, TRUE);
		return 1;
	}

	lua_pushboolean(L, FALSE);
	lua_pushfstring(L, "failed to parse address: %s", addr_str);
	return 2;
}

gpointer
init_fuzzy(struct rspamd_config *cfg)
{
	struct rspamd_fuzzy_storage_ctx *ctx;
	GQuark type;

	type = g_quark_try_string("fuzzy");

	ctx = rspamd_mempool_alloc0(cfg->cfg_pool,
								sizeof(struct rspamd_fuzzy_storage_ctx));

	ctx->magic = rspamd_fuzzy_storage_magic;
	ctx->sync_timeout = DEFAULT_SYNC_TIMEOUT;
	ctx->keypair_cache_size = DEFAULT_KEYPAIR_CACHE_SIZE;
	ctx->keys = kh_init(rspamd_fuzzy_keys_hash);
	rspamd_mempool_add_destructor(cfg->cfg_pool,
								  (rspamd_mempool_destruct_t) fuzzy_hash_table_dtor, ctx->keys);
	ctx->errors_ips = rspamd_lru_hash_new_full(1024,
											   (GDestroyNotify) rspamd_inet_address_free, g_free,
											   rspamd_inet_address_hash, rspamd_inet_address_equal);
	rspamd_mempool_add_destructor(cfg->cfg_pool,
								  (rspamd_mempool_destruct_t) rspamd_lru_hash_destroy, ctx->errors_ips);
	ctx->cfg = cfg;
	ctx->updates_maxfail = DEFAULT_UPDATES_MAXFAIL;
	ctx->leaky_bucket_mask = DEFAULT_BUCKET_MASK;
	ctx->leaky_bucket_ttl = DEFAULT_BUCKET_TTL;
	ctx->max_buckets = DEFAULT_MAX_BUCKETS;
	ctx->leaky_bucket_burst = NAN;
	ctx->leaky_bucket_rate = NAN;
	ctx->delay = NAN;
	ctx->tcp_timeout = DEFAULT_TCP_TIMEOUT;
	ctx->default_forbidden_ids = kh_init(fuzzy_key_ids_set);
	rspamd_mempool_add_destructor(cfg->cfg_pool,
								  (rspamd_mempool_destruct_t) kh_destroy_fuzzy_key_ids_set,
								  ctx->default_forbidden_ids);
	ctx->weak_ids = kh_init(fuzzy_key_ids_set);
	rspamd_mempool_add_destructor(cfg->cfg_pool,
								  (rspamd_mempool_destruct_t) kh_destroy_fuzzy_key_ids_set,
								  ctx->weak_ids);

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "sync",
									  rspamd_rcl_parse_struct_time,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
													  sync_timeout),
									  RSPAMD_CL_FLAG_TIME_FLOAT,
									  "Time to perform database sync, default: " G_STRINGIFY(DEFAULT_SYNC_TIMEOUT) " seconds");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "expire",
									  rspamd_rcl_parse_struct_time,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
													  expire),
									  RSPAMD_CL_FLAG_TIME_FLOAT,
									  "Default expire time for hashes, default: " G_STRINGIFY(DEFAULT_EXPIRE) " seconds");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "delay",
									  rspamd_rcl_parse_struct_time,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
													  delay),
									  RSPAMD_CL_FLAG_TIME_FLOAT,
									  "Default delay time for hashes, default: not enabled");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "tcp_timeout",
									  rspamd_rcl_parse_struct_time,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
													  tcp_timeout),
									  RSPAMD_CL_FLAG_TIME_FLOAT,
									  "TCP connection timeout, default: " G_STRINGIFY(DEFAULT_TCP_TIMEOUT) " seconds");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "allow_update",
									  rspamd_rcl_parse_struct_ucl,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, update_map),
									  0,
									  "Allow modifications from the following IP addresses");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "allow_update_keys",
									  rspamd_rcl_parse_struct_ucl,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, update_keys_map),
									  0,
									  "Allow modifications for those using specific public keys");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "delay_whitelist",
									  rspamd_rcl_parse_struct_ucl,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, delay_whitelist_map),
									  0,
									  "Disable delay check for the following IP addresses");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "keypair",
									  fuzzy_parse_keypair,
									  ctx,
									  0,
									  RSPAMD_CL_FLAG_MULTIPLE,
									  "Encryption keypair (can be repeated for different keys)");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "dynamic_keys_map",
									  rspamd_rcl_parse_struct_ucl,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, dynamic_keys_map),
									  0,
									  "Dynamic encryption keypairs (can be repeated for different keys)");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "forbidden_ids",
									  fuzzy_parse_ids,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, default_forbidden_ids),
									  0,
									  "Deny specific flags by default");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "weak_ids",
									  fuzzy_parse_ids,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, weak_ids),
									  0,
									  "Treat these flags as weak (i.e. they do not overwrite strong flags)");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "keypair_cache_size",
									  rspamd_rcl_parse_struct_integer,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx,
													  keypair_cache_size),
									  RSPAMD_CL_FLAG_UINT,
									  "Size of keypairs cache, default: " G_STRINGIFY(DEFAULT_KEYPAIR_CACHE_SIZE));

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "encrypted_only",
									  rspamd_rcl_parse_struct_boolean,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, encrypted_only),
									  0,
									  "Allow encrypted requests only (and forbid all unknown keys or plaintext requests)");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "dedicated_update_worker",
									  rspamd_rcl_parse_struct_boolean,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, dedicated_update_worker),
									  0,
									  "Use worker 0 for updates only");

	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "read_only",
									  rspamd_rcl_parse_struct_boolean,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, read_only),
									  0,
									  "Work in read only mode");


	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "blocked",
									  rspamd_rcl_parse_struct_ucl,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, blocked_map),
									  0,
									  "Block requests from specific networks");


	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "updates_maxfail",
									  rspamd_rcl_parse_struct_integer,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, updates_maxfail),
									  RSPAMD_CL_FLAG_UINT,
									  "Maximum number of updates to be failed before discarding");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "skip_hashes",
									  rspamd_rcl_parse_struct_ucl,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, skip_map),
									  0,
									  "Skip specific hashes from the map");

	/* Ratelimits */
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "ratelimit_whitelist",
									  rspamd_rcl_parse_struct_ucl,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, ratelimit_whitelist_map),
									  0,
									  "Skip specific addresses from rate limiting");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "ratelimit_max_buckets",
									  rspamd_rcl_parse_struct_integer,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, max_buckets),
									  RSPAMD_CL_FLAG_UINT,
									  "Maximum number of leaky buckets (default: " G_STRINGIFY(DEFAULT_MAX_BUCKETS) ")");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "ratelimit_network_mask",
									  rspamd_rcl_parse_struct_integer,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_mask),
									  RSPAMD_CL_FLAG_UINT,
									  "Network mask to apply for IPv4 rate addresses (default: " G_STRINGIFY(DEFAULT_BUCKET_MASK) ")");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "ratelimit_bucket_ttl",
									  rspamd_rcl_parse_struct_time,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_ttl),
									  RSPAMD_CL_FLAG_TIME_INTEGER,
									  "Time to live for ratelimit element (default: " G_STRINGIFY(DEFAULT_BUCKET_TTL) ")");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "ratelimit_rate",
									  rspamd_rcl_parse_struct_double,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_rate),
									  0,
									  "Leak rate in requests per second");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "ratelimit_burst",
									  rspamd_rcl_parse_struct_double,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, leaky_bucket_burst),
									  0,
									  "Peak value for ratelimit bucket");
	rspamd_rcl_register_worker_option(cfg,
									  type,
									  "ratelimit_log_only",
									  rspamd_rcl_parse_struct_boolean,
									  ctx,
									  G_STRUCT_OFFSET(struct rspamd_fuzzy_storage_ctx, ratelimit_log_only),
									  0,
									  "Don't really ban on ratelimit reaching, just log");


	return ctx;
}

static void
rspamd_fuzzy_peer_io(EV_P_ ev_io *w, int revents)
{
	struct fuzzy_peer_cmd cmd;
	struct rspamd_fuzzy_storage_ctx *ctx =
		(struct rspamd_fuzzy_storage_ctx *) w->data;
	gssize r;

	for (;;) {
		r = read(w->fd, &cmd, sizeof(cmd));

		if (r != sizeof(cmd)) {
			if (errno == EINTR) {
				continue;
			}
			if (errno != EAGAIN) {
				msg_err("cannot read command from peers: %s", strerror(errno));
			}

			break;
		}
		else {
			g_array_append_val(ctx->updates_pending, cmd);
		}
	}
}

static void
fuzzy_peer_rep(struct rspamd_worker *worker,
			   struct rspamd_srv_reply *rep, int rep_fd,
			   gpointer ud)
{
	struct rspamd_fuzzy_storage_ctx *ctx = ud;
	GList *cur;
	struct rspamd_worker_listen_socket *ls;
	struct rspamd_worker_accept_event *ac_ev;

	ctx->peer_fd = rep_fd;

	if (rep_fd == -1) {
		msg_err("cannot receive peer fd from the main process");
		exit(EXIT_FAILURE);
	}
	else {
		rspamd_socket_nonblocking(rep_fd);
	}

	msg_info("got peer fd reply from the main process");

	/* Start listening */
	cur = worker->cf->listen_socks;
	while (cur) {
		ls = cur->data;

		if (ls->fd != -1) {
			msg_info("start listening on %s",
					 rspamd_inet_address_to_string_pretty(ls->addr));

			if (ls->type == RSPAMD_WORKER_SOCKET_UDP) {
				ac_ev = g_malloc0(sizeof(*ac_ev));
				ac_ev->accept_ev.data = worker;
				ac_ev->event_loop = ctx->event_loop;
				ev_io_init(&ac_ev->accept_ev, accept_fuzzy_socket, ls->fd,
						   EV_READ);
				ev_io_start(ctx->event_loop, &ac_ev->accept_ev);
				DL_APPEND(worker->accept_events, ac_ev);
			}
			else if (ls->type == RSPAMD_WORKER_SOCKET_TCP) {
				ac_ev = g_malloc0(sizeof(*ac_ev));
				ac_ev->accept_ev.data = worker;
				ac_ev->event_loop = ctx->event_loop;
				ev_io_init(&ac_ev->accept_ev, accept_tcp_socket, ls->fd,
						   EV_READ);
				ev_io_start(ctx->event_loop, &ac_ev->accept_ev);
				DL_APPEND(worker->accept_events, ac_ev);
			}
			else {
				/* Unknown socket type */
				g_assert_not_reached();
			}
		}

		cur = g_list_next(cur);
	}

	if (ctx->peer_fd != -1) {
		if (worker->index == 0) {
			/* Listen for peer requests */
			shutdown(ctx->peer_fd, SHUT_WR);
			ctx->peer_ev.data = ctx;
			ev_io_init(&ctx->peer_ev, rspamd_fuzzy_peer_io, ctx->peer_fd, EV_READ);
			ev_io_start(ctx->event_loop, &ctx->peer_ev);
		}
		else {
			shutdown(ctx->peer_fd, SHUT_RD);
		}
	}
}

/*
 * Start worker process
 */
__attribute__((noreturn)) void
start_fuzzy(struct rspamd_worker *worker)
{
	struct rspamd_fuzzy_storage_ctx *ctx = worker->ctx;
	GError *err = NULL;
	struct rspamd_srv_command srv_cmd;
	struct rspamd_config *cfg = worker->srv->cfg;

	g_assert(rspamd_worker_check_context(worker->ctx, rspamd_fuzzy_storage_magic));
	ctx->event_loop = rspamd_prepare_worker(worker,
											"fuzzy",
											NULL);
	ctx->peer_fd = -1;
	ctx->worker = worker;
	ctx->cfg = worker->srv->cfg;
	CFG_REF_RETAIN(ctx->cfg);
	ctx->resolver = rspamd_dns_resolver_init(worker->srv->logger,
											 ctx->event_loop,
											 worker->srv->cfg);
	rspamd_upstreams_library_config(worker->srv->cfg, ctx->cfg->ups_ctx,
									ctx->event_loop, ctx->resolver->r);
	/* Since this worker uses maps it needs a valid HTTP context */
	ctx->http_ctx = rspamd_http_context_create(ctx->cfg, ctx->event_loop,
											   ctx->cfg->ups_ctx);

	if (ctx->keypair_cache_size > 0) {
		/* Create keypairs cache */
		ctx->keypair_cache = rspamd_keypair_cache_new(ctx->keypair_cache_size);
	}


	if ((ctx->backend = rspamd_fuzzy_backend_create(ctx->event_loop,
													worker->cf->options, cfg, &err)) == NULL) {
		msg_err("cannot open backend: %e", err);
		if (err) {
			g_error_free(err);
		}
		exit(EXIT_SUCCESS);
	}

	rspamd_fuzzy_backend_count(ctx->backend, fuzzy_count_callback, ctx);


	if (worker->index == 0) {
		ctx->updates_pending = g_array_sized_new(FALSE, FALSE,
												 sizeof(struct fuzzy_peer_cmd), 1024);
		rspamd_fuzzy_backend_start_update(ctx->backend, ctx->sync_timeout,
										  rspamd_fuzzy_storage_periodic_callback, ctx);

		if (ctx->dedicated_update_worker && worker->cf->count > 1) {
			msg_info_config("stop serving clients request in dedicated update mode");
			rspamd_worker_stop_accept(worker);

			GList *cur = worker->cf->listen_socks;

			while (cur) {
				struct rspamd_worker_listen_socket *ls =
					(struct rspamd_worker_listen_socket *) cur->data;

				if (ls->fd != -1) {
					close(ls->fd);
				}

				cur = g_list_next(cur);
			}
		}
	}

	ctx->stat_ev.data = ctx;
	ev_timer_init(&ctx->stat_ev, rspamd_fuzzy_stat_callback, ctx->sync_timeout,
				  ctx->sync_timeout);
	ev_timer_start(ctx->event_loop, &ctx->stat_ev);
	/* Register custom reload and stat commands for the control socket */
	rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_RELOAD,
										  rspamd_fuzzy_storage_reload, ctx);
	rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_FUZZY_STAT,
										  rspamd_fuzzy_storage_stat, ctx);
	rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_FUZZY_HASH,
										  rspamd_fuzzy_storage_hash_info, ctx);
	rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_FUZZY_SYNC,
										  rspamd_fuzzy_storage_sync, ctx);
	rspamd_control_worker_add_cmd_handler(worker, RSPAMD_CONTROL_FUZZY_BLOCKED,
										  rspamd_fuzzy_control_blocked, ctx);


	if (ctx->update_map != NULL) {
		rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->update_map,
									 "Allow fuzzy updates from specified addresses",
									 &ctx->update_ips, NULL, worker, "fuzzy update");
	}

	if (ctx->update_keys_map != NULL) {
		struct rspamd_map *m;

		if ((m = rspamd_map_add_from_ucl(worker->srv->cfg, ctx->update_keys_map,
										 "Allow fuzzy updates from specified public keys",
										 rspamd_kv_list_read,
										 rspamd_kv_list_fin,
										 rspamd_kv_list_dtor,
										 (void **) &ctx->update_keys, worker, RSPAMD_MAP_DEFAULT)) == NULL) {
			msg_warn_config("cannot load allow keys map from %s",
							ucl_object_tostring(ctx->update_keys_map));
		}
		else {
			m->active_http = TRUE;
		}
	}

	if (ctx->skip_map != NULL) {
		struct rspamd_map *m;

		if ((m = rspamd_map_add_from_ucl(cfg, ctx->skip_map,
										 "Skip hashes",
										 rspamd_kv_list_read,
										 rspamd_kv_list_fin,
										 rspamd_kv_list_dtor,
										 (void **) &ctx->skip_hashes,
										 worker, RSPAMD_MAP_DEFAULT)) == NULL) {
			msg_warn_config("cannot load hashes list from %s",
							ucl_object_tostring(ctx->skip_map));
		}
		else {
			m->active_http = TRUE;
		}
	}

	if (ctx->blocked_map != NULL) {
		rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->blocked_map,
									 "Block fuzzy requests from the specific IPs",
									 &ctx->blocked_ips,
									 NULL,
									 worker, "fuzzy blocked");
	}

	/* Create radix trees */
	if (ctx->ratelimit_whitelist_map != NULL) {
		rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->ratelimit_whitelist_map,
									 "Skip ratelimits from specific ip addresses/networks",
									 &ctx->ratelimit_whitelist,
									 NULL,
									 worker, "fuzzy ratelimit whitelist");
	}

	if (ctx->dynamic_keys_map) {
		struct fuzzy_keymap_ucl_buf *jb, **pjb;

		ctx->dynamic_keys = kh_init(rspamd_fuzzy_keys_hash);
		rspamd_mempool_add_destructor(ctx->cfg->cfg_pool,
									  (rspamd_mempool_destruct_t) fuzzy_hash_table_dtor,
									  ctx->dynamic_keys);
		/* Now try to add map with ucl data */
		jb = g_malloc(sizeof(struct fuzzy_keymap_ucl_buf));
		pjb = g_malloc(sizeof(struct fuzzy_keymap_ucl_buf *));
		jb->buf = NULL;
		jb->ctx = ctx;
		*pjb = jb;
		rspamd_mempool_add_destructor(ctx->cfg->cfg_pool,
									  (rspamd_mempool_destruct_t) g_free,
									  pjb);

		if (!rspamd_map_add_from_ucl(cfg,
									 ctx->dynamic_keys_map,
									 "Dynamic fuzzy keys map",
									 ucl_keymap_read_cb,
									 ucl_keymap_fin_cb,
									 ucl_keymap_dtor_cb,
									 (void **) pjb, worker, RSPAMD_MAP_DEFAULT)) {
			msg_err("cannot add map for dynamic keys");
		}
	}

	if (!isnan(ctx->delay) && ctx->delay_whitelist_map != NULL) {
		rspamd_config_radix_from_ucl(worker->srv->cfg, ctx->delay_whitelist_map,
									 "Skip delay from the following ips",
									 &ctx->delay_whitelist, NULL, worker,
									 "fuzzy delayed whitelist");
	}

	/* Ratelimits */
	ctx->ratelimit_buckets = rspamd_lru_hash_new_full(ctx->max_buckets,
													  NULL, fuzzy_rl_bucket_free,
													  rspamd_inet_address_hash, rspamd_inet_address_equal);
	ctx->dynamic_blocked_nets = radix_create_compressed("dynamic_blocked_nets");

	rspamd_fuzzy_maybe_load_ratelimits(ctx);

	/* Maps events */
	ctx->resolver = rspamd_dns_resolver_init(worker->srv->logger,
											 ctx->event_loop,
											 worker->srv->cfg);
	rspamd_map_watch(worker->srv->cfg, ctx->event_loop,
					 ctx->resolver, worker, RSPAMD_MAP_WATCH_WORKER);

	/* Get peer pipe */
	memset(&srv_cmd, 0, sizeof(srv_cmd));
	srv_cmd.type = RSPAMD_SRV_SOCKETPAIR;
	srv_cmd.cmd.spair.af = SOCK_DGRAM;
	srv_cmd.cmd.spair.pair_num = worker->index;
	memset(srv_cmd.cmd.spair.pair_id, 0, sizeof(srv_cmd.cmd.spair.pair_id));
	/* 6 bytes of id (including \0) and bind_conf id */
	G_STATIC_ASSERT(sizeof(srv_cmd.cmd.spair.pair_id) >=
					sizeof("fuzzy") + sizeof(uint64_t));

	memcpy(srv_cmd.cmd.spair.pair_id, "fuzzy", sizeof("fuzzy"));

	/* Distinguish workers from each others... */
	if (worker->cf->bind_conf && worker->cf->bind_conf->bind_line) {
		uint64_t bind_hash = rspamd_cryptobox_fast_hash(worker->cf->bind_conf->bind_line,
														strlen(worker->cf->bind_conf->bind_line), 0xdeadbabe);

		/* 8 more bytes */
		memcpy(srv_cmd.cmd.spair.pair_id + sizeof("fuzzy"), &bind_hash,
			   sizeof(bind_hash));
	}

	rspamd_srv_send_command(worker, ctx->event_loop, &srv_cmd, -1,
							fuzzy_peer_rep, ctx);

	/*
	 * Extra fields available for this particular worker
	 */
	luaL_Reg fuzzy_lua_reg = {
		.name = "add_fuzzy_pre_handler",
		.func = lua_fuzzy_add_pre_handler,
	};
	rspamd_lua_add_metamethod(ctx->cfg->lua_state, rspamd_worker_classname, &fuzzy_lua_reg);
	fuzzy_lua_reg = (luaL_Reg) {
		.name = "add_fuzzy_post_handler",
		.func = lua_fuzzy_add_post_handler,
	};
	rspamd_lua_add_metamethod(ctx->cfg->lua_state, rspamd_worker_classname, &fuzzy_lua_reg);
	fuzzy_lua_reg = (luaL_Reg) {
		.name = "add_fuzzy_blacklist_handler",
		.func = lua_fuzzy_add_blacklist_handler,
	};
	rspamd_lua_add_metamethod(ctx->cfg->lua_state, rspamd_worker_classname, &fuzzy_lua_reg);
	fuzzy_lua_reg = (luaL_Reg) {
		.name = "block_fuzzy_client",
		.func = lua_fuzzy_block_client,
	};
	rspamd_lua_add_metamethod(ctx->cfg->lua_state, rspamd_worker_classname, &fuzzy_lua_reg);
	fuzzy_lua_reg = (luaL_Reg) {
		.name = "is_ratelimit_whitelisted",
		.func = lua_fuzzy_is_ratelimit_whitelisted,
	};
	rspamd_lua_add_metamethod(ctx->cfg->lua_state, rspamd_worker_classname, &fuzzy_lua_reg);

	rspamd_lua_run_postloads(ctx->cfg->lua_state, ctx->cfg, ctx->event_loop,
							 worker);

	ev_loop(ctx->event_loop, 0);
	rspamd_worker_block_signals();

	if (ctx->peer_fd != -1) {
		if (worker->index == 0) {
			ev_io_stop(ctx->event_loop, &ctx->peer_ev);
		}
		close(ctx->peer_fd);
	}

	if (worker->index == 0 && ctx->updates_pending->len > 0) {

		msg_info_config("start another event loop to sync fuzzy storage");

		if (rspamd_fuzzy_process_updates_queue(ctx, local_db_name, TRUE)) {
			ev_loop(ctx->event_loop, 0);
			msg_info_config("sync cycle is done");
		}
		else {
			msg_info_config("no need to sync");
		}
	}

	rspamd_fuzzy_backend_close(ctx->backend);

	if (worker->index == 0) {
		g_array_free(ctx->updates_pending, TRUE);
		ctx->updates_pending = NULL;
	}

	if (ctx->keypair_cache) {
		rspamd_keypair_cache_destroy(ctx->keypair_cache);
	}

	if (ctx->ratelimit_buckets) {
		/* Try the best to save ratelimits from the proper worker */
		if ((!ctx->dedicated_update_worker && worker->index == 0) ||
			(ctx->dedicated_update_worker && worker->index == 1)) {
			rspamd_fuzzy_maybe_save_ratelimits(ctx);
		}

		rspamd_lru_hash_destroy(ctx->ratelimit_buckets);
	}

	if (ctx->dynamic_blocked_nets) {
		/* Ban structs are pool-allocated inside the radix tree's pool,
		 * so radix_destroy_compressed frees them all at once */
		radix_destroy_compressed(ctx->dynamic_blocked_nets);
		ctx->dynamic_blocked_nets = NULL;
	}

	/* Drain any peer-pipe write requests that were waiting for the write
	 * watcher to fire. The event loop has already broken out of ev_loop on
	 * non-update workers, so these watchers will never fire on their own and
	 * the up_req allocations would leak. */
	{
		struct fuzzy_peer_request *pr, *pr_tmp;
		DL_FOREACH_SAFE(ctx->pending_peer_requests, pr, pr_tmp)
		{
			if (ev_can_stop(&pr->io_ev)) {
				ev_io_stop(ctx->event_loop, &pr->io_ev);
			}
			DL_DELETE(ctx->pending_peer_requests, pr);
			g_free(pr);
		}
	}

	struct rspamd_lua_fuzzy_script *cur, *tmp;

	LL_FOREACH_SAFE(ctx->lua_pre_handlers, cur, tmp)
	{
		luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref);
		g_free(cur);
	}
	LL_FOREACH_SAFE(ctx->lua_post_handlers, cur, tmp)
	{
		luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref);
		g_free(cur);
	}
	LL_FOREACH_SAFE(ctx->lua_blacklist_handlers, cur, tmp)
	{
		luaL_unref(ctx->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref);
		g_free(cur);
	}

	/* default_forbidden_ids and weak_ids are freed by mempool destructor */

	CFG_REF_RELEASE(ctx->cfg);
	CFG_REF_RELEASE(ctx->cfg);
	rspamd_log_close(worker->srv->logger);
	rspamd_unset_crash_handler(worker->srv);

	exit(EXIT_SUCCESS);
}
