diff --git a/pkg/util/user_agent.go b/pkg/util/user_agent.go index 5355ca34..5e996409 100644 --- a/pkg/util/user_agent.go +++ b/pkg/util/user_agent.go @@ -5,14 +5,25 @@ import "strings" // IsSocialMediaBot checks if the User-Agent belongs to a social media crawler. func IsSocialMediaBot(ua string) bool { bots := []string{ + // Meta (Facebook, Instagram, WhatsApp) "facebookexternalhit", + "facebookcatalog", + "facebot", + "meta-externalagent", + // Twitter/X "twitterbot", + // LinkedIn "linkedinbot", - "whatsapp", - "slackbot", + // Discord "discordbot", + // Telegram "telegrambot", - "facebot", + // Slack + "slackbot", + // WhatsApp + "whatsapp", + // Pinterest + "pinterestbot", } ua = strings.ToLower(ua) for _, bot := range bots {